[Runtime] Use GIL lock in the runtime instead of custom mutexes. (#624)

**Context:** We have a custom mutex that denotes current use of the
Python interpreter (a shared resource among threads). This custom mutex
may be replaced by the lock found in pybind11.

**Description of the Change:** Use the scope acquire lock.

**Benefits:** No custom mutex, better DX with third party libraries. (No
need to determine location of the mutex library).

**Possible Drawbacks:**

**Related GitHub Issues:**

TODO:

- [ ] ~~add a python file and outline all strings that are input to
exec~~
- [x] OQC device
- [x] release gil when executing the memory transfer
- [x] write functions and classes for dlopen dlclose

---------

Co-authored-by: Joey Carter <[email protected]>

.github/workflows/check-catalyst.yaml

@@ -638,10 +638,11 @@ jobs:
       - name: Build Runtime test suite for OpenQasm device
         if: ${{ matrix.backend == 'openqasm' }}
         run: |
+          # Asan prevents dlopen from working?
           C_COMPILER=$(which ${{ needs.constants.outputs[format('c_compiler.{0}', matrix.compiler)] }}) \
           CXX_COMPILER=$(which ${{ needs.constants.outputs[format('cxx_compiler.{0}', matrix.compiler)] }}) \
           COMPILER_LAUNCHER="" \
-          ENABLE_ASAN=ON \
+          ENABLE_ASAN=OFF \
           make test-runtime
 
       - name: Build examples

.github/workflows/scripts/linux_arm64/rh8/build_catalyst.sh

@@ -56,7 +56,6 @@ cmake --build runtime-build --target rt_capi rtd_lightning rtd_openqasm rtd_null
 # Build OQC
 export OQC_BUILD_DIR="/catalyst/oqc-build"
 export RT_BUILD_DIR="/catalyst/runtime-build"
-export USE_ALTERNATIVE_CATALYST_PYTHON_INTERPRETER=ON
 make oqc
 
 # Build Catalyst dialects
@@ -80,6 +79,7 @@ cmake --build quantum-build --target check-dialects compiler_driver catalyst-cli
 # Copy files needed for the wheel where they are expected
 cp /catalyst/runtime-build/lib/*/*/*/*/librtd* /catalyst/runtime-build/lib
 cp /catalyst/runtime-build/lib/registry/runtime-build/lib/catalyst_callback_registry.cpython-${PYTHON_ALTERNATIVE_VERSION}-aarch64-linux-gnu.so /catalyst/runtime-build/lib
+cp /catalyst/runtime-build/lib/*/*/*/*/openqasm_python_module.cpython-${PYTHON_ALTERNATIVE_VERSION}-aarch64-linux-gnu.so /catalyst/runtime-build/lib
 cp /catalyst/runtime-build/lib/capi/runtime-build/lib/librt_capi.so /catalyst/runtime-build/lib/
 
 # Build wheels

Makefile

@@ -171,6 +171,7 @@ wheel:
 	mkdir -p $(MK_DIR)/frontend/catalyst/lib/backend
 	cp $(RT_BUILD_DIR)/lib/librtd* $(MK_DIR)/frontend/catalyst/lib
 	cp $(RT_BUILD_DIR)/lib/catalyst_callback_registry*.* $(MK_DIR)/frontend/catalyst/lib
+	cp $(RT_BUILD_DIR)/lib/openqasm_python_module*.* $(MK_DIR)/frontend/catalyst/lib
 	cp $(RT_BUILD_DIR)/lib/librt_capi.* $(MK_DIR)/frontend/catalyst/lib
 	cp $(RT_BUILD_DIR)/lib/backend/*.toml $(MK_DIR)/frontend/catalyst/lib/backend
 	cp $(OQC_BUILD_DIR)/librtd_oqc* $(MK_DIR)/frontend/catalyst/lib

frontend/catalyst/third_party/oqc/src/CMakeLists.txt

@@ -2,14 +2,10 @@ cmake_minimum_required(VERSION 3.20)
 
 project(catalyst_oqc)
 
-option(USE_ALTERNATIVE_CATALYST_PYTHON_INTERPRETER_PATH OFF)
-
 set(runtime_includes "${PROJECT_SOURCE_DIR}/../../../../../runtime/include")
 set(backend_includes "${PROJECT_SOURCE_DIR}/../../../../../runtime/lib/backend/common")
-set(util_includes "${PROJECT_SOURCE_DIR}/../../../../../runtime/utils")
 set(runtime_lib "${RUNTIME_BUILD_DIR}/lib")
 set(oqc_backend_dir "${OQC_BUILD_DIR}/backend")
-set(catalyst_python_interpreter_path "${RUNTIME_BUILD_DIR}/utils/runtime-build/lib")
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -47,24 +43,29 @@ add_library(rtd_oqc SHARED OQCDevice.cpp)
 target_include_directories(rtd_oqc PUBLIC .
     ${runtime_includes}
     ${backend_includes}
-    ${util_includes}
     )
 
 set(OQC_LIBRARIES
     rtd_oqc
-    catalyst_python_interpreter
 )
 
 set_target_properties(rtd_oqc PROPERTIES BUILD_RPATH "$ORIGIN/../utils")
 target_link_directories(rtd_oqc PRIVATE ${runtime_lib})
-target_link_libraries(rtd_oqc PRIVATE pybind11::module catalyst_python_interpreter)
 
-if(USE_ALTERNATIVE_CATALYST_PYTHON_INTERPRETER_PATH)
- target_link_directories(rtd_oqc PRIVATE  ${catalyst_python_interpreter_path})
-endif()
-unset(USE_ALTERNATIVE_CATALYST_PYTHON_INTERPRETER_PATH CACHE)
+pybind11_add_module(oqc_python_module SHARED oqc_python_module.cpp)
+target_include_directories(oqc_python_module PRIVATE ${runtime_includes})
+
+add_dependencies(rtd_oqc oqc_python_module)
+target_compile_definitions(rtd_oqc PUBLIC -DOQC_PY=\"$<TARGET_FILE_NAME:oqc_python_module>\")
 
 set_property(TARGET rtd_oqc PROPERTY POSITION_INDEPENDENT_CODE ON)
+set_property(TARGET rtd_oqc APPEND PROPERTY BUILD_RPATH "$<TARGET_FILE_DIR:oqc_python_module>")
+if(NOT APPLE)
+    set_property(TARGET rtd_oqc APPEND PROPERTY BUILD_RPATH $ORIGIN)
+else()
+    set_property(TARGET rtd_oqc APPEND PROPERTY BUILD_RPATH @loader_path)
+endif()
+
 file(COPY ${PROJECT_SOURCE_DIR}/oqc.toml DESTINATION ./backend)
 
 add_subdirectory(tests)

frontend/catalyst/third_party/oqc/src/Makefile

@@ -7,7 +7,6 @@ MK_ABSPATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 MK_DIR := $(dir $(MK_ABSPATH))
 OQC_BUILD_DIR?=$(MK_DIR)/build
 RT_BUILD_DIR?=$(MK_DIR)/../../../../../runtime/build
-USE_ALTERNATIVE_CATALYST_PYTHON_INTERPRETER?=OFF
 
 .PHONY: configure
 configure:
@@ -17,7 +16,6 @@ configure:
 		-DCMAKE_C_COMPILER=$(C_COMPILER) \
 		-DCMAKE_CXX_COMPILER=$(CXX_COMPILER) \
 		-DRUNTIME_BUILD_DIR=$(RT_BUILD_DIR) \
-		-DUSE_ALTERNATIVE_CATALYST_PYTHON_INTERPRETER_PATH=$(USE_ALTERNATIVE_CATALYST_PYTHON_INTERPRETER) \
 		-DPYTHON_EXECUTABLE=$(PYTHON) \
 		-Dpybind11_DIR=$(shell $(PYTHON) -c "import pybind11; print(pybind11.get_cmake_dir())")
 

frontend/catalyst/third_party/oqc/src/OQCDevice.hpp

@@ -31,9 +31,7 @@
 #include "QubitManager.hpp"
 #include "Utils.hpp"
 
-#include <pybind11/embed.h>
-
-#include "OQCRunner.hpp" // <pybind11/embed.h>
+#include "OQCRunner.hpp"
 #include "OpenQASM2Builder.hpp"
 
 using namespace Catalyst::Runtime::OpenQASM2;

frontend/catalyst/third_party/oqc/src/OQCRunner.hpp

@@ -16,19 +16,15 @@
 #pragma once
 
 #include <complex>
-#include <mutex>
-#include <optional>
 #include <string>
-#include <unordered_map>
-#include <utility>
 #include <vector>
 
-#include <iostream>
-
+#include "DynamicLibraryLoader.hpp"
 #include "Exception.hpp"
-#include "Python.hpp"
 
+#ifdef INITIALIZE_PYTHON
 #include <pybind11/embed.h>
+#endif
 
 namespace Catalyst::Runtime::Device {
 
@@ -119,54 +115,19 @@ struct OQCRunner : public OQCRunnerBase {
                               size_t num_qubits, const std::string &kwargs = "") const
         -> std::vector<size_t>
     {
-        std::lock_guard<std::mutex> lock(getPythonMutex());
-        namespace py = pybind11;
-        using namespace py::literals;
-
-        RT_FAIL_IF(!Py_IsInitialized(), "The Python interpreter is not initialized");
-
-        auto locals = py::dict("circuit"_a = circuit, "device"_a = device, "kwargs"_a = kwargs,
-                               "shots"_a = shots, "msg"_a = "");
-
-        py::exec(
-            R"(
-            import os
-            from qcaas_client.client import OQCClient, QPUTask, CompilerConfig
-            from qcaas_client.config import QuantumResultsFormat, Tket, TketOptimizations
-            optimisations = Tket()
-            optimisations.tket_optimizations = TketOptimizations.DefaultMappingPass
-
-            RES_FORMAT = QuantumResultsFormat().binary_count()
-
-            try:
-                email = os.environ.get("OQC_EMAIL")
-                password = os.environ.get("OQC_PASSWORD")
-                url = os.environ.get("OQC_URL")
-                client = OQCClient(url=url, email=email, password=password)
-                client.authenticate()
-                oqc_config = CompilerConfig(repeats=shots, results_format=RES_FORMAT, optimizations=optimisations)
-                oqc_task = QPUTask(circuit, oqc_config)
-                res = client.execute_tasks(oqc_task)
-                counts = res[0].result["cbits"]
-
-            except Exception as e:
-                print(f"circuit: {circuit}")
-                msg = str(e)
-            )",
-            py::globals(), locals);
-
-        auto &&msg = locals["msg"].cast<std::string>();
-        RT_FAIL_IF(!msg.empty(), msg.c_str());
-
-        py::dict results = locals["counts"];
-
-        std::vector<size_t> counts_value;
-        for (auto item : results) {
-            auto key = item.first;
-            auto value = item.second;
-            counts_value.push_back(value.cast<size_t>());
+#ifdef INITIALIZE_PYTHON
+        if (!Py_IsInitialized()) {
+            pybind11::initialize_interpreter();
         }
-        return counts_value;
+#endif
+
+        DynamicLibraryLoader libLoader(OQC_PY);
+
+        using countsImpl_t =
+            std::vector<size_t> (*)(const char *, const char *, size_t, const char *);
+        auto countsImpl = libLoader.getSymbol<countsImpl_t>("counts");
+
+        return countsImpl(circuit.c_str(), device.c_str(), shots, kwargs.c_str());
     }
 };
 

frontend/catalyst/third_party/oqc/src/oqc_python_module.cpp

@@ -0,0 +1,73 @@
+// Copyright 2024 Xanadu Quantum Technologies Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <pybind11/eval.h>
+#include <pybind11/pybind11.h>
+#include <string>
+
+#include "Exception.hpp"
+
+std::string program = R"(
+import os
+from qcaas_client.client import OQCClient, QPUTask, CompilerConfig
+from qcaas_client.config import QuantumResultsFormat, Tket, TketOptimizations
+optimisations = Tket()
+optimisations.tket_optimizations = TketOptimizations.DefaultMappingPass
+
+RES_FORMAT = QuantumResultsFormat().binary_count()
+
+try:
+    email = os.environ.get("OQC_EMAIL")
+    password = os.environ.get("OQC_PASSWORD")
+    url = os.environ.get("OQC_URL")
+    client = OQCClient(url=url, email=email, password=password)
+    client.authenticate()
+    oqc_config = CompilerConfig(repeats=shots, results_format=RES_FORMAT, optimizations=optimisations)
+    oqc_task = QPUTask(circuit, oqc_config)
+    res = client.execute_tasks(oqc_task)
+    counts = res[0].result["cbits"]
+
+except Exception as e:
+    print(f"circuit: {circuit}")
+    msg = str(e)
+)";
+
+[[gnu::visibility("default")]] void counts(const char *_circuit, const char *_device, size_t shots,
+                                           size_t num_qubits, const char *_kwargs, void *_vector)
+{
+    namespace py = pybind11;
+    using namespace py::literals;
+
+    py::gil_scoped_acquire lock;
+
+    auto locals = py::dict("circuit"_a = _circuit, "device"_a = _device, "kwargs"_a = _kwargs,
+                           "shots"_a = shots, "msg"_a = "");
+
+    py::exec(program, py::globals(), locals);
+
+    auto &&msg = locals["msg"].cast<std::string>();
+    RT_FAIL_IF(!msg.empty(), msg.c_str());
+
+    py::dict results = locals["counts"];
+
+    std::vector<size_t> *counts_value = reinterpret_cast<std::vector<size_t> *>(_vector);
+    for (auto item : results) {
+        auto key = item.first;
+        auto value = item.second;
+        counts_value->push_back(value.cast<size_t>());
+    }
+    return;
+}
+
+PYBIND11_MODULE(oqc_python_module, m) { m.doc() = "oqc"; }

frontend/catalyst/third_party/oqc/src/tests/CMakeLists.txt

@@ -24,13 +24,23 @@ include(CTest)
 include(Catch)
 
 add_executable(runner_tests_oqc runner_main.cpp)
+add_dependencies(runner_tests_oqc oqc_python_module)
+set_property(TARGET runner_tests_oqc APPEND PROPERTY BUILD_RPATH "$<TARGET_FILE_DIR:oqc_python_module>")
+if(NOT APPLE)
+    set_property(TARGET runner_tests_oqc APPEND PROPERTY BUILD_RPATH $ORIGIN)
+    set_property(TARGET runner_tests_oqc APPEND PROPERTY BUILD_RPATH $ORIGIN/../lib)
+else()
+    set_property(TARGET runner_tests_oqc APPEND PROPERTY BUILD_RPATH @loader_path)
+    set_property(TARGET runner_tests_oqc APPEND PROPERTY BUILD_RPATH @loader_path/../lib)
+endif()
 
 target_include_directories(runner_tests_oqc PRIVATE
     ${OQC_LIBRARIES}
     )
 
 target_link_directories(runner_tests_oqc PRIVATE ${runtime_lib})
 # To avoid link to libpython, we use pybind11::module interface library.
+target_compile_definitions(runner_tests_oqc PUBLIC INITIALIZE_PYTHON)
 target_link_libraries(runner_tests_oqc PRIVATE
     Catch2::Catch2
     pybind11::embed

frontend/catalyst/third_party/oqc/src/tests/Test_OQCDevice.cpp

@@ -14,11 +14,8 @@
 
 #include "OQCDevice.cpp"
 #include "OQCRunner.hpp"
-#include "Python.hpp"
 #include "RuntimeCAPI.h"
 
-PythonInterpreterGuard guard{};
-
 #include <catch2/catch.hpp>
 
 using namespace Catalyst::Runtime::Device;
@@ -108,4 +105,4 @@ TEST_CASE("Test the bell pair circuit", "[openqasm]")
                               "h qubits[2];\n";
 
     CHECK(device->Circuit() == toqasmempty);
-}
+}

frontend/catalyst/utils/wrapper.cpp

@@ -214,7 +214,10 @@ py::list wrap(py::object func, py::tuple py_args, py::object result_desc, py::ob
     auto value1 = py_args.attr("__getitem__")(1);
     void *value1_ptr = *reinterpret_cast<void **>(ctypes.attr("addressof")(value1).cast<size_t>());
 
-    f_ptr(value0_ptr, value1_ptr);
+    {
+        py::gil_scoped_release lock;
+        f_ptr(value0_ptr, value1_ptr);
+    }
     returns = move_returns(value0_ptr, result_desc, transfer, numpy_arrays);
 
     return returns;

runtime/CMakeLists.txt

@@ -25,7 +25,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(runtime_includes "${PROJECT_SOURCE_DIR}/include")
 set(capi_utils_includes "${PROJECT_SOURCE_DIR}/lib/capi")
 set(backend_includes "${PROJECT_SOURCE_DIR}/lib/backend/common")
-set(util_includes "${PROJECT_SOURCE_DIR}/utils")
 
 
 # Get LLVM hash to target from source tree.
@@ -130,4 +129,3 @@ endif()
 
 add_subdirectory(lib)
 add_subdirectory(tests)
-add_subdirectory(utils)