From 7e47114da651cb8a33ba9a9aeabfa06f9a06237d Mon Sep 17 00:00:00 2001 From: Alexander Condello Date: Thu, 3 Nov 2022 13:52:16 -0700 Subject: [PATCH] Experiment with using taskflow for parallel presolving --- .circleci/config.yml | 12 +++++++++++ .gitmodules | 3 +++ MANIFEST.in | 2 ++ dwave/preprocessing/include/dwave/presolve.h | 21 +++++++++++++++++++- dwave/preprocessing/libcpp.pxd | 1 + dwave/preprocessing/presolve/cypresolve.pyx | 7 +++++++ extern/taskflow | 1 + setup.py | 7 ++++--- tests/test_presolve.py | 10 ++++++++++ testscpp/Makefile | 9 +++++---- testscpp/tests/test_presolve.cpp | 10 ++++++++++ 11 files changed, 75 insertions(+), 8 deletions(-) create mode 160000 extern/taskflow diff --git a/.circleci/config.yml b/.circleci/config.yml index 14278bf..474cded 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,6 +22,8 @@ jobs: steps: - checkout + - run: git submodule sync + - run: git submodule update --init - setup_remote_docker - restore_cache: &build-linux-restore-cache keys: @@ -61,6 +63,8 @@ jobs: steps: &build-steps - checkout + - run: git submodule sync + - run: git submodule update --init - restore_cache: *build-linux-restore-cache - run: *build-linux-wheels - save_cache: *build-linux-save-cache @@ -90,6 +94,8 @@ jobs: steps: - checkout + - run: git submodule sync + - run: git submodule update --init - run: name: build sdist command: | @@ -115,6 +121,8 @@ jobs: steps: - checkout + - run: git submodule sync + - run: git submodule update --init - run: name: build wheels command: | @@ -152,6 +160,8 @@ jobs: steps: - checkout + - run: git submodule sync + - run: git submodule update --init - run: name: install dependencies command: | @@ -181,6 +191,8 @@ jobs: steps: - checkout + - run: git submodule sync + - run: git submodule update --init - run: name: install doxygen command: sudo apt-get install doxygen diff --git a/.gitmodules b/.gitmodules index 51341cc..ab8fdf6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = testscpp/Catch2 url = https://github.com/catchorg/Catch2.git branch = v2.x +[submodule "extern/taskflow"] + path = extern/taskflow + url = git@github.com:taskflow/taskflow.git diff --git a/MANIFEST.in b/MANIFEST.in index e7418e2..d37a65b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include pyproject.toml recursive-include dwave/preprocessing/include/ *.hpp *.h recursive-include dwave/preprocessing *.pyx *.pxd *.pyx.src +graft extern/taskflow/taskflow +include extern/taskflow/LICENSE diff --git a/dwave/preprocessing/include/dwave/presolve.h b/dwave/preprocessing/include/dwave/presolve.h index fc97f4a..2240244 100644 --- a/dwave/preprocessing/include/dwave/presolve.h +++ b/dwave/preprocessing/include/dwave/presolve.h @@ -14,12 +14,17 @@ #pragma once +#include +#include + + #include #include #include #include -#include "dimod/constrained_quadratic_model.h" +#include +#include namespace dwave { namespace presolve { @@ -134,10 +139,24 @@ class Presolver { /// Apply any loaded presolve techniques. Acts of the model() in-place. void apply(); + void apply_parallel() { + tf::Executor executor; + tf::Taskflow taskflow; + + taskflow.for_each_index(0, static_cast(model_.num_constraints()), 1, + [&](int i) { do_slow_thing(i); }); + + executor.run(taskflow).wait(); + } + /// Detach the constrained quadratic model and return it. /// This clears the model from the presolver. model_type detach_model(); + void do_slow_thing(index_type c) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + /// Load the default presolve techniques. void load_default_presolvers(); diff --git a/dwave/preprocessing/libcpp.pxd b/dwave/preprocessing/libcpp.pxd index 25c9e0f..7020675 100644 --- a/dwave/preprocessing/libcpp.pxd +++ b/dwave/preprocessing/libcpp.pxd @@ -29,6 +29,7 @@ cdef extern from "dwave/presolve.h" namespace "dwave::presolve" nogil: Presolver() Presolver(model_type) void apply() except+ + void apply_parallel() except+ model_type detach_model() void load_default_presolvers() model_type& model() diff --git a/dwave/preprocessing/presolve/cypresolve.pyx b/dwave/preprocessing/presolve/cypresolve.pyx index 55481e4..8478a8b 100644 --- a/dwave/preprocessing/presolve/cypresolve.pyx +++ b/dwave/preprocessing/presolve/cypresolve.pyx @@ -47,6 +47,13 @@ cdef class cyPresolver: self.cpppresolver.apply() self._model_num_variables = self.cpppresolver.model().num_variables() + def apply_parallel(self): + import time + + t = time.perf_counter() + self.cpppresolver.apply_parallel() + return time.perf_counter() - t + def clear_model(self): """Clear the held model. This is useful to save memory.""" self.cpppresolver.detach_model() diff --git a/extern/taskflow b/extern/taskflow new file mode 160000 index 0000000..f1490ff --- /dev/null +++ b/extern/taskflow @@ -0,0 +1 @@ +Subproject commit f1490ffc286eba418107c4aa9b8913e8ca76443c diff --git a/setup.py b/setup.py index 2db315f..1fd745c 100644 --- a/setup.py +++ b/setup.py @@ -23,13 +23,13 @@ from distutils.command.build_ext import build_ext as _build_ext extra_compile_args = { - 'msvc': ['/EHsc'], - 'unix': ['-std=c++11'], + 'msvc': ['/std:c++17', '/EHsc'], + 'unix': ['-std=c++17', '-pthread'], } extra_link_args = { 'msvc': [], - 'unix': ['-std=c++11'], + 'unix': ['-std=c++17'], } @@ -60,6 +60,7 @@ def build_extensions(self): include_dirs=[ numpy.get_include(), dimod.get_include(), + "extern/taskflow/", ], install_requires=[ 'numpy>=1.20.0,<2.0.0', # keep synced with circle-ci, pyproject.toml diff --git a/tests/test_presolve.py b/tests/test_presolve.py index 7b5e229..65f5a9d 100644 --- a/tests/test_presolve.py +++ b/tests/test_presolve.py @@ -20,6 +20,16 @@ from dwave.preprocessing import Presolver, InfeasibleModelError +class TestExperiment(unittest.TestCase): + def test_parallel(self): + cqm = dimod.CQM() + for _ in range(100): + cqm.add_constraint(dimod.BQM("BINARY") == 1) + + presolver = Presolver(cqm) + self.assertLessEqual(presolver.apply_parallel(), 55) # should take 100s + + class TestPresolver(unittest.TestCase): def test_bug0(self): random = np.random.RandomState(0) diff --git a/testscpp/Makefile b/testscpp/Makefile index 5213ba2..7e9dc7a 100644 --- a/testscpp/Makefile +++ b/testscpp/Makefile @@ -2,6 +2,7 @@ ROOT := ../ SRC := $(ROOT)/dwave/preprocessing/ CATCH2 := $(ROOT)/testscpp/Catch2/single_include/ DIMOD := $(shell python -c 'import dimod; print(dimod.get_include())') +TASKFLOW := $(ROOT)/extern/taskflow/ all: catch2 test_main test_main_parallel tests tests_parallel @@ -12,12 +13,12 @@ tests_parallel: test_main_parallel.out ./test_main_parallel test_main: test_main.cpp - g++ -std=c++11 -Wall -c test_main.cpp - g++ -std=c++11 -Wall test_main.o tests/*.cpp -o test_main -I $(SRC)/include/ -I $(DIMOD) -I $(CATCH2) + g++ -std=c++17 -Wall -pthread -c test_main.cpp + g++ -std=c++17 -Wall -pthread test_main.o tests/*.cpp -o test_main -I $(SRC)/include/ -I $(DIMOD) -I $(CATCH2) -I $(TASKFLOW) test_main_parallel: test_main.cpp - g++ -std=c++11 -fopenmp -Wall -c test_main.cpp -o test_main_parallel.o - g++ -std=c++11 -fopenmp -Wall test_main_parallel.o tests/*.cpp -o test_main_parallel -I $(SRC)/include/ -I $(DIMOD) -I $(CATCH2) + g++ -std=c++17 -fopenmp -Wall -c test_main.cpp -o test_main_parallel.o + g++ -std=c++17 -fopenmp -Wall test_main_parallel.o tests/*.cpp -o test_main_parallel -I $(SRC)/include/ -I $(DIMOD) -I $(CATCH2) -I $(TASKFLOW) catch2: git submodule init diff --git a/testscpp/tests/test_presolve.cpp b/testscpp/tests/test_presolve.cpp index fcceabe..f523abd 100644 --- a/testscpp/tests/test_presolve.cpp +++ b/testscpp/tests/test_presolve.cpp @@ -350,4 +350,14 @@ SCENARIO("constrained quadratic models can be presolved") { } } +TEST_CASE("experiment") { + GIVEN("a CQM with a bunch of constraints") { + auto cqm = dimod::ConstrainedQuadraticModel(); + cqm.add_constraints(10); + + auto presolver = presolve::Presolver(std::move(cqm)); + presolver.apply_parallel(); + } +} + } // namespace dwave