Add Python C++ pybind basic structure

gpu/CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.18)
+
+project(alluxioGPU)
+
+include(cmake/Utils.cmake)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+# Include directories
+include_directories(
+  "${PROJECT_SOURCE_DIR}"
+  "${PROJECT_SOURCE_DIR}/cpp"
+  "${PROJECT_BINARY_DIR}")
+
+################################################
+# Build Alluxio GPU libraries
+################################################
+add_subdirectory(cpp)
+add_subdirectory(python)

gpu/README.md

@@ -0,0 +1,19 @@
+# GPU Prototype
+
+## Build Process
+
+Done it once:
+```
+pip install pybind11
+```
+
+```
+export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:/opt/anaconda3/lib/python3.11/site-packages/pybind11/share/cmake/pybind11
+```
+
+```
+python setup.py bdist_wheel
+export MACOSX_DEPLOYMENT_TARGET=10.9 # may not be necessary for you
+pip install dist/dist/alluxio_gpu-0.1.0-cp311-cp311-macosx_10_9_x86_64.whl
+pytest tests/test_example.py
+```

gpu/cmake/Utils.cmake

@@ -0,0 +1,58 @@
+include(FetchContent)
+
+FetchContent_Declare(
+  pybind11
+  GIT_REPOSITORY "https://github.com/pybind/pybind11.git"
+  GIT_TAG "v2.12.0"
+)
+FetchContent_MakeAvailable(pybind11)
+
+# Helper function to remove elements from a variable
+function(remove TARGET INPUT)
+  foreach(ITEM ${ARGN})
+    list(REMOVE_ITEM INPUT "${ITEM}")
+  endforeach()
+  set(${TARGET} ${INPUT} PARENT_SCOPE)
+endfunction(remove)
+
+# Collect headers in the current directory
+macro(collect_headers HEADERS_GROUP)
+  cmake_parse_arguments(
+    COLLECT_HEADERS
+    "PARENT_SCOPE;INCLUDE_TEST"
+    ""
+    ""
+    ${ARGV})
+
+  file(GLOB collect_headers_tmp *.h *.hpp *.cuh *.inl)
+  set(${HEADERS_GROUP} ${${HEADERS_GROUP}} ${collect_headers_tmp})
+
+  # We remove filenames containing substring "test"
+  if(NOT COLLECT_HEADERS_INCLUDE_TEST)
+    file(GLOB collect_headers_tmp *test*)
+    remove(${HEADERS_GROUP} "${${HEADERS_GROUP}}" ${collect_headers_tmp})
+  endif()
+
+  if(COLLECT_HEADERS_PARENT_SCOPE)
+    set(${HEADERS_GROUP} ${${HEADERS_GROUP}} PARENT_SCOPE)
+  endif()
+endmacro(collect_headers)
+
+# Collect sources in the current directory
+macro(collect_sources SRCS_GROUP)
+  cmake_parse_arguments(
+    COLLECT_SOURCES
+    "PARENT_SCOPE"
+    ""
+    ""
+    ${ARGV})
+
+  file(GLOB collect_sources_tmp *.cc *.cu *.c)
+  file(GLOB collect_sources_tmp_test *_test.cc *_test.cu *_test.c)
+  remove(collect_sources_tmp "${collect_sources_tmp}" ${collect_sources_tmp_test})
+  set(${SRCS_GROUP} ${${SRCS_GROUP}} ${collect_sources_tmp})
+
+  if (COLLECT_SOURCES_PARENT_SCOPE)
+    set(${SRCS_GROUP} ${${SRCS_GROUP}} PARENT_SCOPE)
+  endif()
+endmacro(collect_sources)

gpu/cpp/CMakeLists.txt

@@ -0,0 +1,8 @@
+# Collect sources and headers for the cpp library
+collect_sources(CORE_SRCS PARENT_SCOPE)
+collect_headers(CORE_HDRS PARENT_SCOPE)
+
+## Why here is core, can it be others
+add_library(cpp SHARED ${CORE_SRCS})
+set_target_properties(cpp PROPERTIES PREFIX "")
+target_include_directories(cpp PRIVATE ${CMAKE_SOURCE_DIR}/cpp)

gpu/cpp/example.cc

@@ -0,0 +1,5 @@
+#include "example.h"
+
+int add(int a, int b) {
+    return a + b;
+}

gpu/cpp/example.h

@@ -0,0 +1,6 @@
+#ifndef EXAMPLE_H
+#define EXAMPLE_H
+
+int add(int a, int b);
+
+#endif // EXAMPLE_H

gpu/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel",
+    "ninja",
+    "cmake>=3.12",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.mypy]
+files = "setup.py"
+python_version = "3.7"
+strict = true
+show_error_codes = true
+enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
+warn_unreachable = true
+
+[[tool.mypy.overrides]]
+module = ["ninja"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
+xfail_strict = true
+filterwarnings = [
+    "error",
+    "ignore:(ast.Str|Attribute s|ast.NameConstant|ast.Num) is deprecated:DeprecationWarning:_pytest",
+]
+testpaths = ["tests"]
+
+[tool.cibuildwheel]
+test-command = "pytest {project}/tests"
+test-extras = ["test"]
+test-skip = ["*universal2:arm64"]
+before-build = "rm -rf {project}/build"
+
+[tool.ruff]
+target-version = "py37"
+
+[tool.ruff.lint]
+extend-select = [
+  "B",    # flake8-bugbear
+  "I",    # isort
+  "PGH",  # pygrep-hooks
+  "RUF",  # Ruff-specific
+  "UP",   # pyupgrade
+]

gpu/python/CMakeLists.txt

@@ -0,0 +1,8 @@
+find_package(pybind11 REQUIRED)
+
+collect_sources(PYTHON_SRCS)
+collect_headers(PYTHON_HEADERS)
+
+pybind11_add_module(example_py example_py.cc)
+set_target_properties(example_py PROPERTIES PREFIX "")
+target_link_libraries(example_py PRIVATE cpp)

gpu/python/example_py.cc

@@ -0,0 +1,8 @@
+#include <pybind11/pybind11.h>
+#include "example.h"
+
+namespace py = pybind11;
+
+PYBIND11_MODULE(example_py, m) {
+    m.def("add", &add, "A function that adds two numbers");
+}

gpu/setup.py

@@ -0,0 +1,116 @@
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+from setuptools import Extension
+from setuptools import setup
+from setuptools.command.build_ext import build_ext
+
+# Convert distutils Windows platform specifiers to CMake -A arguments
+PLAT_TO_CMAKE = {
+    "win32": "Win32",
+    "win-amd64": "x64",
+    "win-arm32": "ARM",
+    "win-arm64": "ARM64",
+}
+
+
+class CMakeExtension(Extension):
+    def __init__(self, name: str, sourcedir: str = "") -> None:
+        super().__init__(name, sources=[])
+        self.sourcedir = os.fspath(Path(sourcedir).resolve())
+
+
+class CMakeBuild(build_ext):
+    def build_extension(self, ext: CMakeExtension) -> None:
+        ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name)
+        extdir = ext_fullpath.parent.resolve()
+
+        debug = (
+            int(os.environ.get("DEBUG", 0))
+            if self.debug is None
+            else self.debug
+        )
+        cfg = "Debug" if debug else "Release"
+
+        cmake_generator = os.environ.get("CMAKE_GENERATOR", "")
+
+        cmake_args = [
+            f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}",
+            f"-DPYTHON_EXECUTABLE={sys.executable}",
+            f"-DCMAKE_BUILD_TYPE={cfg}",
+        ]
+        build_args = []
+
+        if "CMAKE_ARGS" in os.environ:
+            cmake_args += [
+                item for item in os.environ["CMAKE_ARGS"].split(" ") if item
+            ]
+
+        # cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"]
+
+        if self.compiler.compiler_type != "msvc":
+            if not cmake_generator or cmake_generator == "Ninja":
+                try:
+                    import ninja
+
+                    ninja_executable_path = Path(ninja.BIN_DIR) / "ninja"
+                    cmake_args += [
+                        "-GNinja",
+                        f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}",
+                    ]
+                except ImportError:
+                    pass
+        else:
+            single_config = any(
+                x in cmake_generator for x in {"NMake", "Ninja"}
+            )
+            contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"})
+
+            if not single_config and not contains_arch:
+                cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]]
+
+            if not single_config:
+                cmake_args += [
+                    f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"
+                ]
+                build_args += ["--config", cfg]
+
+        if sys.platform.startswith("darwin"):
+            archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", ""))
+            if archs:
+                cmake_args += [
+                    "-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))
+                ]
+
+        if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ:
+            if hasattr(self, "parallel") and self.parallel:
+                build_args += [f"-j{self.parallel}"]
+
+        build_temp = Path(self.build_temp) / ext.name
+        if not build_temp.exists():
+            build_temp.mkdir(parents=True)
+
+        subprocess.run(
+            ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True
+        )
+        subprocess.run(
+            ["cmake", "--build", ".", *build_args], cwd=build_temp, check=True
+        )
+
+
+setup(
+    name="alluxio_gpu",
+    version="0.1.0",
+    author="Your Name",
+    author_email="[email protected]",
+    description="An example project with C++ and Python extensions",
+    long_description="This project demonstrates building a Python package with C++ extensions using CMake.",
+    ext_modules=[CMakeExtension("example_py", sourcedir=".")],
+    cmdclass={"build_ext": CMakeBuild},
+    zip_safe=False,
+    extras_require={"test": ["pytest>=6.0"]},
+    python_requires=">=3.7",
+)

gpu/tests/test_example.py

@@ -0,0 +1,7 @@
+import example_py
+
+
+def test_add():
+    assert example_py.add(1, 2) == 3
+    assert example_py.add(-1, 1) == 0
+    assert example_py.add(-1, -1) == -2