diff --git a/gpu/CMakeLists.txt b/gpu/CMakeLists.txt new file mode 100644 index 0000000..7852215 --- /dev/null +++ b/gpu/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.18) + +project(alluxioGPU) + +include(cmake/Utils.cmake) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# Include directories +include_directories( + "${PROJECT_SOURCE_DIR}" + "${PROJECT_SOURCE_DIR}/cpp" + "${PROJECT_BINARY_DIR}") + +################################################ +# Build Alluxio GPU libraries +################################################ +add_subdirectory(cpp) +add_subdirectory(python) diff --git a/gpu/README.md b/gpu/README.md new file mode 100644 index 0000000..dd63a8e --- /dev/null +++ b/gpu/README.md @@ -0,0 +1,19 @@ +# GPU Prototype + +## Build Process + +Done it once: +``` +pip install pybind11 +``` + +``` +export CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH:/opt/anaconda3/lib/python3.11/site-packages/pybind11/share/cmake/pybind11 +``` + +``` +python setup.py bdist_wheel +export MACOSX_DEPLOYMENT_TARGET=10.9 # may not be necessary for you +pip install dist/dist/alluxio_gpu-0.1.0-cp311-cp311-macosx_10_9_x86_64.whl +pytest tests/test_example.py +``` diff --git a/gpu/cmake/Utils.cmake b/gpu/cmake/Utils.cmake new file mode 100644 index 0000000..c2ba2d6 --- /dev/null +++ b/gpu/cmake/Utils.cmake @@ -0,0 +1,58 @@ +include(FetchContent) + +FetchContent_Declare( + pybind11 + GIT_REPOSITORY "https://github.com/pybind/pybind11.git" + GIT_TAG "v2.12.0" +) +FetchContent_MakeAvailable(pybind11) + +# Helper function to remove elements from a variable +function(remove TARGET INPUT) + foreach(ITEM ${ARGN}) + list(REMOVE_ITEM INPUT "${ITEM}") + endforeach() + set(${TARGET} ${INPUT} PARENT_SCOPE) +endfunction(remove) + +# Collect headers in the current directory +macro(collect_headers HEADERS_GROUP) + cmake_parse_arguments( + COLLECT_HEADERS + "PARENT_SCOPE;INCLUDE_TEST" + "" + "" + ${ARGV}) + + file(GLOB collect_headers_tmp *.h *.hpp *.cuh *.inl) + set(${HEADERS_GROUP} ${${HEADERS_GROUP}} ${collect_headers_tmp}) + + # We remove filenames containing substring "test" + if(NOT COLLECT_HEADERS_INCLUDE_TEST) + file(GLOB collect_headers_tmp *test*) + remove(${HEADERS_GROUP} "${${HEADERS_GROUP}}" ${collect_headers_tmp}) + endif() + + if(COLLECT_HEADERS_PARENT_SCOPE) + set(${HEADERS_GROUP} ${${HEADERS_GROUP}} PARENT_SCOPE) + endif() +endmacro(collect_headers) + +# Collect sources in the current directory +macro(collect_sources SRCS_GROUP) + cmake_parse_arguments( + COLLECT_SOURCES + "PARENT_SCOPE" + "" + "" + ${ARGV}) + + file(GLOB collect_sources_tmp *.cc *.cu *.c) + file(GLOB collect_sources_tmp_test *_test.cc *_test.cu *_test.c) + remove(collect_sources_tmp "${collect_sources_tmp}" ${collect_sources_tmp_test}) + set(${SRCS_GROUP} ${${SRCS_GROUP}} ${collect_sources_tmp}) + + if (COLLECT_SOURCES_PARENT_SCOPE) + set(${SRCS_GROUP} ${${SRCS_GROUP}} PARENT_SCOPE) + endif() +endmacro(collect_sources) diff --git a/gpu/cpp/CMakeLists.txt b/gpu/cpp/CMakeLists.txt new file mode 100644 index 0000000..ea8dcd8 --- /dev/null +++ b/gpu/cpp/CMakeLists.txt @@ -0,0 +1,8 @@ +# Collect sources and headers for the cpp library +collect_sources(CORE_SRCS PARENT_SCOPE) +collect_headers(CORE_HDRS PARENT_SCOPE) + +## Why here is core, can it be others +add_library(cpp SHARED ${CORE_SRCS}) +set_target_properties(cpp PROPERTIES PREFIX "") +target_include_directories(cpp PRIVATE ${CMAKE_SOURCE_DIR}/cpp) diff --git a/gpu/cpp/example.cc b/gpu/cpp/example.cc new file mode 100644 index 0000000..2c2cfd4 --- /dev/null +++ b/gpu/cpp/example.cc @@ -0,0 +1,5 @@ +#include "example.h" + +int add(int a, int b) { + return a + b; +} diff --git a/gpu/cpp/example.h b/gpu/cpp/example.h new file mode 100644 index 0000000..5edf5b6 --- /dev/null +++ b/gpu/cpp/example.h @@ -0,0 +1,6 @@ +#ifndef EXAMPLE_H +#define EXAMPLE_H + +int add(int a, int b); + +#endif // EXAMPLE_H diff --git a/gpu/pyproject.toml b/gpu/pyproject.toml new file mode 100644 index 0000000..00f0008 --- /dev/null +++ b/gpu/pyproject.toml @@ -0,0 +1,48 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "ninja", + "cmake>=3.12", +] +build-backend = "setuptools.build_meta" + +[tool.mypy] +files = "setup.py" +python_version = "3.7" +strict = true +show_error_codes = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +warn_unreachable = true + +[[tool.mypy.overrides]] +module = ["ninja"] +ignore_missing_imports = true + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] +xfail_strict = true +filterwarnings = [ + "error", + "ignore:(ast.Str|Attribute s|ast.NameConstant|ast.Num) is deprecated:DeprecationWarning:_pytest", +] +testpaths = ["tests"] + +[tool.cibuildwheel] +test-command = "pytest {project}/tests" +test-extras = ["test"] +test-skip = ["*universal2:arm64"] +before-build = "rm -rf {project}/build" + +[tool.ruff] +target-version = "py37" + +[tool.ruff.lint] +extend-select = [ + "B", # flake8-bugbear + "I", # isort + "PGH", # pygrep-hooks + "RUF", # Ruff-specific + "UP", # pyupgrade +] diff --git a/gpu/python/CMakeLists.txt b/gpu/python/CMakeLists.txt new file mode 100644 index 0000000..a717de9 --- /dev/null +++ b/gpu/python/CMakeLists.txt @@ -0,0 +1,8 @@ +find_package(pybind11 REQUIRED) + +collect_sources(PYTHON_SRCS) +collect_headers(PYTHON_HEADERS) + +pybind11_add_module(example_py example_py.cc) +set_target_properties(example_py PROPERTIES PREFIX "") +target_link_libraries(example_py PRIVATE cpp) diff --git a/gpu/python/example_py.cc b/gpu/python/example_py.cc new file mode 100644 index 0000000..cd26c0e --- /dev/null +++ b/gpu/python/example_py.cc @@ -0,0 +1,8 @@ +#include +#include "example.h" + +namespace py = pybind11; + +PYBIND11_MODULE(example_py, m) { + m.def("add", &add, "A function that adds two numbers"); +} diff --git a/gpu/setup.py b/gpu/setup.py new file mode 100644 index 0000000..b95ff01 --- /dev/null +++ b/gpu/setup.py @@ -0,0 +1,116 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +from setuptools import Extension +from setuptools import setup +from setuptools.command.build_ext import build_ext + +# Convert distutils Windows platform specifiers to CMake -A arguments +PLAT_TO_CMAKE = { + "win32": "Win32", + "win-amd64": "x64", + "win-arm32": "ARM", + "win-arm64": "ARM64", +} + + +class CMakeExtension(Extension): + def __init__(self, name: str, sourcedir: str = "") -> None: + super().__init__(name, sources=[]) + self.sourcedir = os.fspath(Path(sourcedir).resolve()) + + +class CMakeBuild(build_ext): + def build_extension(self, ext: CMakeExtension) -> None: + ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) + extdir = ext_fullpath.parent.resolve() + + debug = ( + int(os.environ.get("DEBUG", 0)) + if self.debug is None + else self.debug + ) + cfg = "Debug" if debug else "Release" + + cmake_generator = os.environ.get("CMAKE_GENERATOR", "") + + cmake_args = [ + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}", + f"-DPYTHON_EXECUTABLE={sys.executable}", + f"-DCMAKE_BUILD_TYPE={cfg}", + ] + build_args = [] + + if "CMAKE_ARGS" in os.environ: + cmake_args += [ + item for item in os.environ["CMAKE_ARGS"].split(" ") if item + ] + + # cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"] + + if self.compiler.compiler_type != "msvc": + if not cmake_generator or cmake_generator == "Ninja": + try: + import ninja + + ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" + cmake_args += [ + "-GNinja", + f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", + ] + except ImportError: + pass + else: + single_config = any( + x in cmake_generator for x in {"NMake", "Ninja"} + ) + contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) + + if not single_config and not contains_arch: + cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] + + if not single_config: + cmake_args += [ + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" + ] + build_args += ["--config", cfg] + + if sys.platform.startswith("darwin"): + archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", "")) + if archs: + cmake_args += [ + "-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs)) + ] + + if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: + if hasattr(self, "parallel") and self.parallel: + build_args += [f"-j{self.parallel}"] + + build_temp = Path(self.build_temp) / ext.name + if not build_temp.exists(): + build_temp.mkdir(parents=True) + + subprocess.run( + ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True + ) + subprocess.run( + ["cmake", "--build", ".", *build_args], cwd=build_temp, check=True + ) + + +setup( + name="alluxio_gpu", + version="0.1.0", + author="Your Name", + author_email="your.email@example.com", + description="An example project with C++ and Python extensions", + long_description="This project demonstrates building a Python package with C++ extensions using CMake.", + ext_modules=[CMakeExtension("example_py", sourcedir=".")], + cmdclass={"build_ext": CMakeBuild}, + zip_safe=False, + extras_require={"test": ["pytest>=6.0"]}, + python_requires=">=3.7", +) diff --git a/gpu/tests/test_example.py b/gpu/tests/test_example.py new file mode 100644 index 0000000..6e5073d --- /dev/null +++ b/gpu/tests/test_example.py @@ -0,0 +1,7 @@ +import example_py + + +def test_add(): + assert example_py.add(1, 2) == 3 + assert example_py.add(-1, 1) == 0 + assert example_py.add(-1, -1) == -2