[WIP] replace new compiler driver by calling compiler-cli using subpr…

…ocess (#1285)

**Context:** The old compiler driver api which uses the pybind11 as an
interface between the python frontend and the compiler should be
replaced with the new CLI tool (catalyst-cli).

**Description of the Change:**
Removes the pybind11 interface in favour of calling catalyst-cli using
python subprocess.

**Benefits:**
the catalyst-cli will be completely independent of the python frontend
and consequently, compiler symbols will be separated from the frontend

**Possible Drawbacks:**
Using subprocess requires us to connect with the catalyst-cli using
files on disk. Therefore we need to save the IR to a file temporarily
and parse it again in the compiler which can be problematic when IR size
grows. This should be improved in future using interprocess
communication.

[sc-73537]
[sc-73538]

.github/workflows/build-wheel-linux-x86_64.yaml

@@ -379,7 +379,7 @@ jobs:
               -DLLVM_ENABLE_ZSTD=OFF \
               -DLLVM_ENABLE_LLD=ON
 
-        cmake --build quantum-build --target check-dialects compiler_driver catalyst-cli
+        cmake --build quantum-build --target check-dialects catalyst-cli
 
     - name: Build wheel
       run: |

.github/workflows/build-wheel-macos-arm64.yaml

@@ -344,7 +344,7 @@ jobs:
               -DLLVM_ENABLE_LLD=OFF \
               -DLLVM_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm
 
-        cmake --build quantum-build --target check-dialects compiler_driver catalyst-cli
+        cmake --build quantum-build --target check-dialects catalyst-cli
 
     - name: Build wheel
       run: |

.github/workflows/build-wheel-macos-x86_64.yaml

@@ -334,7 +334,7 @@ jobs:
               -DLLVM_ENABLE_ZSTD=OFF \
               -DLLVM_ENABLE_LLD=OFF
 
-        cmake --build quantum-build --target check-dialects compiler_driver catalyst-cli
+        cmake --build quantum-build --target check-dialects catalyst-cli
 
     - name: Build wheel
       run: |

.github/workflows/check-catalyst.yaml

@@ -440,6 +440,8 @@ jobs:
         echo "PYTHONPATH=$PYTHONPATH:$(pwd)/quantum-build/python_packages/quantum" >> $GITHUB_ENV
         echo "RUNTIME_LIB_DIR=$(pwd)/runtime-build/lib" >> $GITHUB_ENV
         echo "MLIR_LIB_DIR=$(pwd)/llvm-build/lib" >> $GITHUB_ENV
+        echo "CATALYST_BIN_DIR=$(pwd)/quantum-build/bin" >> $GITHUB_ENV
+        chmod +x $(pwd)/quantum-build/bin/catalyst-cli  # artifact upload does not preserve permissions
 
     - name: Run Python Lit Tests
       run: |
@@ -511,6 +513,8 @@ jobs:
         echo "PYTHONPATH=$PYTHONPATH:$(pwd)/quantum-build/python_packages/quantum" >> $GITHUB_ENV
         echo "RUNTIME_LIB_DIR=$(pwd)/runtime-build/lib" >> $GITHUB_ENV
         echo "MLIR_LIB_DIR=$(pwd)/llvm-build/lib" >> $GITHUB_ENV
+        echo "CATALYST_BIN_DIR=$(pwd)/quantum-build/bin" >> $GITHUB_ENV
+        chmod +x $(pwd)/quantum-build/bin/catalyst-cli  # artifact upload does not preserve permissions
 
     - name: Run Python Pytest Tests (backend=lightning.kokkos)
       run: |
@@ -566,6 +570,8 @@ jobs:
         echo "PYTHONPATH=$PYTHONPATH:$(pwd)/quantum-build/python_packages/quantum" >> $GITHUB_ENV
         echo "RUNTIME_LIB_DIR=$(pwd)/runtime-build/lib" >> $GITHUB_ENV
         echo "MLIR_LIB_DIR=$(pwd)/llvm-build/lib" >> $GITHUB_ENV
+        echo "CATALYST_BIN_DIR=$(pwd)/quantum-build/bin" >> $GITHUB_ENV
+        chmod +x $(pwd)/quantum-build/bin/catalyst-cli  # artifact upload does not preserve permissions
 
     - name: Run Python Pytest Tests
       run: |

.github/workflows/check-pl-compat.yaml

@@ -194,6 +194,8 @@ jobs:
         echo "PYTHONPATH=$PYTHONPATH:$(pwd)/quantum-build/python_packages/quantum" >> $GITHUB_ENV
         echo "RUNTIME_LIB_DIR=$(pwd)/runtime-build/lib" >> $GITHUB_ENV
         echo "MLIR_LIB_DIR=$(pwd)/llvm-build/lib" >> $GITHUB_ENV
+        echo "CATALYST_BIN_DIR=$(pwd)/quantum-build/bin" >> $GITHUB_ENV
+        chmod +x $(pwd)/quantum-build/bin/catalyst-cli  # artifact upload does not preserve permissions
 
     - name: Run Frontend Tests
       run: |

.github/workflows/scripts/linux_arm64/rh8/build_catalyst.sh

@@ -74,7 +74,7 @@ cmake -S mlir -B quantum-build -G Ninja \
     -DLLVM_ENABLE_ZSTD=OFF \
     -DLLVM_ENABLE_LLD=ON \
     -DLLVM_DIR=/catalyst/llvm-build/lib/cmake/llvm
-cmake --build quantum-build --target check-dialects compiler_driver catalyst-cli
+cmake --build quantum-build --target check-dialects catalyst-cli
 
 # Copy files needed for the wheel where they are expected
 cp /catalyst/runtime-build/lib/*/*/*/*/librtd* /catalyst/runtime-build/lib

Makefile

@@ -187,7 +187,8 @@ wheel:
 	for file in gradient quantum _ods_common catalyst mitigation _transform; do \
 		cp $(COPY_FLAGS) $(DIALECTS_BUILD_DIR)/python_packages/quantum/mlir_quantum/dialects/*$${file}* $(MK_DIR)/frontend/mlir_quantum/dialects ; \
 	done
-	cp $(COPY_FLAGS) $(DIALECTS_BUILD_DIR)/python_packages/quantum/mlir_quantum/compiler_driver.so $(MK_DIR)/frontend/mlir_quantum/
+	mkdir -p $(MK_DIR)/frontend/catalyst/bin
+	cp $(COPY_FLAGS) $(DIALECTS_BUILD_DIR)/bin/catalyst-cli $(MK_DIR)/frontend/catalyst/bin
 	find $(MK_DIR)/frontend -type d -name __pycache__ -exec rm -rf {} +
 
 	$(PYTHON) -m pip wheel --no-deps . -w dist

doc/catalyst-cli/catalyst-cli.rst

@@ -26,8 +26,8 @@ each stage individually. For example:
     # Only lowers LLVM dialect input to LLVM IR
     catalyst-cli --tool=translate llvm-dialect.mlir -o llvm-ir.ll
 
-    # Only performs lower-level optimizations and creates object file
-    catalyst-cli --tool=llc llvm-ir.ll -o output.o
+    # Only performs lower-level optimizations and creates object file (object.o)
+    catalyst-cli --tool=llc llvm-ir.ll -o output.ll --module-name object
 
 .. note::
 
@@ -46,9 +46,8 @@ Usage
 
 Calling ``catalyst-cli`` without any options runs the three compilation stages (``quantum-opt``,
 ``mlir-translate`` and ``llc``) using all default configurations, and outputs by default an object
-file named ``catalyst_module.o``. The name of the output file can be set directly using the ``-o``
-option, or by changing the output module name using the ``--module-name`` option (the default module
-name is ``catalyst_module``).
+file named ``catalyst_module.o``. The name of the output file can be set by changing the output 
+module name using the ``--module-name`` option (the default module name is ``catalyst_module``).
 
 Command line options
 ^^^^^^^^^^^^^^^^^^^^
@@ -70,8 +69,7 @@ Emit verbose messages.
 ``-o <filename>``
 """""""""""""""""
 
-Output filename. If no output filename is provided, and if the ``llc`` compilation step is not run
-to produce an object file, the resulting IR is output to stdout.
+Output IR filename. If no output filename is provided, the resulting IR is output to stdout.
 
 ``--tool=<opt|translate|llc|all>``
 """"""""""""""""""""""""""""""""""
@@ -186,7 +184,7 @@ we can do so as follows:
     pipe(remove-chained-self-inverse{func-name=my_circuit};merge-rotations{func-name=my_circuit})
 
 Finally, we'll use the option ``--mlir-print-ir-after-all`` to print the resulting MLIR after each
-pass that is applied, and the ``-o`` option to set the name of the output file:
+pass that is applied, and the ``-o`` option to set the name of the output IR file:
 
 .. code-block::
 

doc/conf.py

@@ -98,7 +98,6 @@ def __getattr__(cls, name):
     "mlir_quantum.dialects.catalyst",
     "mlir_quantum.dialects.mitigation",
     "mlir_quantum.dialects._transform_ops_gen",
-    "mlir_quantum.compiler_driver",
     "pybind11",
     "cudaq",
 ]

doc/releases/changelog-dev.md

@@ -1,11 +1,13 @@
-:orphan:
-
 # Release 0.10.0-dev (development release)
 
 <h3>New features since last release</h3>
 
 <h3>Improvements 🛠</h3>
 
+* Catalyst now uses the new compiler API to compile quantum code from the python frontend.
+  Frontend no longer uses pybind11 to connect to the compiler. Instead, it uses subprocess instead.
+  [(#1285)](https://github.com/PennyLaneAI/catalyst/pull/1285)
+
 * Replace pybind11 with nanobind for C++/Python bindings in the frontend.
   [(#1173)](https://github.com/PennyLaneAI/catalyst/pull/1173)
 
@@ -43,4 +45,5 @@ This release contains contributions from (in alphabetical order):
 Joey Carter,
 William Maxwell
 Romain Moyard,
-Paul Haochen Wang.
+Paul Haochen Wang,
+Mehrdad Malekmohammadi.