From 62933e06b0cc6e34ebb5c00864cbe407dffb347f Mon Sep 17 00:00:00 2001 From: sameeul Date: Fri, 11 Aug 2023 07:50:12 -0400 Subject: [PATCH 1/4] Update pyarrow linking process --- CMakeLists.txt | 24 ++++++------------------ ci-utils/find_pyarrow.py | 13 ------------- src/nyx/python/nyxus/nyxus.py | 18 +++++++++++------- tests/CMakeLists.txt | 2 +- 4 files changed, 18 insertions(+), 39 deletions(-) delete mode 100644 ci-utils/find_pyarrow.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d604b9d..c518790a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ if(CUDA_FOUND AND USEGPU) set(CUDA_ARCH_LIST "35;37;50;72;75;80;86") endif() elseif (CUDA_VERSION_MAJOR STREQUAL "12") - set(CUDA_ARCH_LIST "52;60;72;75;80;86;89;90") + set(CUDA_ARCH_LIST "52;60;75;80;86;89;90") else() # some old CUDA version (<10) set(CUDA_ARCH_LIST "50") endif() @@ -317,49 +317,37 @@ if(USEARROW) if (BUILD_LIB) # Find Python find_package(Python REQUIRED COMPONENTS Interpreter) - # Execute the Python script to find pyarrow include path execute_process( COMMAND ${Python_EXECUTABLE} -c "import pyarrow as pa; print(pa.get_include())" OUTPUT_VARIABLE PATH_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE ) - # Store the path in a CMake variable set(PYARROW_INCLUDE_PATH "${PATH_OUTPUT}" CACHE INTERNAL "Path to pyarrow include directory") - - # Set filename for finding pyarrow - set(python_filename "ci-utils/find_pyarrow.py") - set(python_filepath "${CMAKE_CURRENT_SOURCE_DIR}/${python_filename}") - # Execute the Python script to find pyarrow library path execute_process( - COMMAND ${Python_EXECUTABLE} ${python_filepath} + COMMAND ${Python_EXECUTABLE} -c "import pyarrow as pa; pa.get_library_dirs()[0]" OUTPUT_VARIABLE LIB_PATH_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE ) - # Store the path in a CMake variable set(PYARROW_LIB_PATH "${LIB_PATH_OUTPUT}" CACHE INTERNAL "Path to pyarrow library directory") - if (PYARROW_INCLUDE_PATH STREQUAL "") message(FATAL_ERROR "Pyarrow library not found.") endif() - if (PYARROW_LIB_PATH STREQUAL "") message(FATAL_ERROR "Pyarrow library not found.") endif() - # Print the path message(STATUS "PyArrow Include Path: ${PYARROW_INCLUDE_PATH}") - message(STATUS "PyArrow Include Path: ${PYARROW_LIB_PATH}") + message(STATUS "PyArrow library Path: ${PYARROW_LIB_PATH}") # add pyarrow include path include_directories(${PYARROW_INCLUDE_PATH}) - - target_link_libraries(backend PRIVATE ${PYARROW_LIB_PATH} ) - - target_link_libraries(backend PRIVATE ${Python_INCLUDE_DIRS}) + #add pyarrow library + list(APPEND Nyxus_LIBRARIES arrow_python) + target_link_directories(backend PRIVATE ${PYARROW_LIB_PATH}) endif() diff --git a/ci-utils/find_pyarrow.py b/ci-utils/find_pyarrow.py deleted file mode 100644 index 362e9067..00000000 --- a/ci-utils/find_pyarrow.py +++ /dev/null @@ -1,13 +0,0 @@ -import pyarrow as pa -import platform -path=pa.get_library_dirs()[0] -operating_system = platform.system() - -if operating_system == 'Linux': - path += '/libarrow_python.so' -elif operating_system == 'Darwin': - path += '/libarrow_python.dylib' -elif operating_system == 'Windows': - path += '\\arrow_python.lib' - -print(path) \ No newline at end of file diff --git a/src/nyx/python/nyxus/nyxus.py b/src/nyx/python/nyxus/nyxus.py index ff26e631..d929c5f3 100644 --- a/src/nyx/python/nyxus/nyxus.py +++ b/src/nyx/python/nyxus/nyxus.py @@ -1,3 +1,14 @@ +import os +import numpy as np +import pandas as pd +from typing import Optional, List + +#pa.import_pyarrow() +import pyarrow as pa +if os.sys.platform == "win32": + for lib_dir in pa.get_library_dirs(): + os.add_dll_directory(lib_dir) + from .backend import ( initialize_environment, featurize_directory_imp, @@ -21,13 +32,6 @@ arrow_is_enabled_imp, ) -import os -import numpy as np -import pandas as pd -import pyarrow as pa -from typing import Optional, List - -#pa.import_pyarrow() class Nyxus: """Nyxus image feature extraction library diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8fa31636..9e2623bf 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -26,7 +26,7 @@ if(CUDA_FOUND AND USEGPU) set(CUDA_ARCH_LIST "35;37;50;72;75;80;86") endif() elseif (CUDA_VERSION_MAJOR STREQUAL "12") - set(CUDA_ARCH_LIST "52;60;72;75;80;86;89;90") + set(CUDA_ARCH_LIST "52;60;75;80;86;89;90") else() # some old CUDA version (<10) set(CUDA_ARCH_LIST "50") endif() From c62bef6b6ed862314a72d0215bcbe1520124051c Mon Sep 17 00:00:00 2001 From: sameeul Date: Fri, 11 Aug 2023 08:23:35 -0400 Subject: [PATCH 2/4] minor fixes --- CMakeLists.txt | 4 ++-- src/nyx/python/nyxus/nyxus.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c518790a..5ab35ee0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -327,7 +327,7 @@ if(USEARROW) set(PYARROW_INCLUDE_PATH "${PATH_OUTPUT}" CACHE INTERNAL "Path to pyarrow include directory") # Execute the Python script to find pyarrow library path execute_process( - COMMAND ${Python_EXECUTABLE} -c "import pyarrow as pa; pa.get_library_dirs()[0]" + COMMAND ${Python_EXECUTABLE} -c "import pyarrow as pa; print(pa.get_library_dirs()[0])" OUTPUT_VARIABLE LIB_PATH_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE ) @@ -341,7 +341,7 @@ if(USEARROW) endif() # Print the path message(STATUS "PyArrow Include Path: ${PYARROW_INCLUDE_PATH}") - message(STATUS "PyArrow library Path: ${PYARROW_LIB_PATH}") + message(STATUS "PyArrow Library Path: ${PYARROW_LIB_PATH}") # add pyarrow include path include_directories(${PYARROW_INCLUDE_PATH}) diff --git a/src/nyx/python/nyxus/nyxus.py b/src/nyx/python/nyxus/nyxus.py index d929c5f3..5da22fb8 100644 --- a/src/nyx/python/nyxus/nyxus.py +++ b/src/nyx/python/nyxus/nyxus.py @@ -3,7 +3,6 @@ import pandas as pd from typing import Optional, List -#pa.import_pyarrow() import pyarrow as pa if os.sys.platform == "win32": for lib_dir in pa.get_library_dirs(): From ae002223f48384aa1683c9e5b2146111c880269c Mon Sep 17 00:00:00 2001 From: sameeul Date: Fri, 11 Aug 2023 09:23:43 -0400 Subject: [PATCH 3/4] fix for Python 3.7 --- src/nyx/python/nyxus/nyxus.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/nyx/python/nyxus/nyxus.py b/src/nyx/python/nyxus/nyxus.py index 5da22fb8..712fd3b6 100644 --- a/src/nyx/python/nyxus/nyxus.py +++ b/src/nyx/python/nyxus/nyxus.py @@ -1,12 +1,18 @@ import os +import sys import numpy as np import pandas as pd from typing import Optional, List -import pyarrow as pa -if os.sys.platform == "win32": - for lib_dir in pa.get_library_dirs(): - os.add_dll_directory(lib_dir) +if sys.version_info[0]==3 and sys.version_info[1]>=8: + # since add_dll_dir is added in Python3.8 + # Assuming this is OK since 3.7 has reached EOL and + # this added search_path is needed on conda distribution only which + # currently builds for 3.8+ + import pyarrow as pa + if os.sys.platform == "win32": + for lib_dir in pa.get_library_dirs(): + os.add_dll_directory(lib_dir) from .backend import ( initialize_environment, From 18022c66fced749f0e60afa95f64f7aef7b8b49a Mon Sep 17 00:00:00 2001 From: sameeul Date: Fri, 11 Aug 2023 09:41:10 -0400 Subject: [PATCH 4/4] cleaner fix --- src/nyx/python/nyxus/nyxus.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/nyx/python/nyxus/nyxus.py b/src/nyx/python/nyxus/nyxus.py index 712fd3b6..99773386 100644 --- a/src/nyx/python/nyxus/nyxus.py +++ b/src/nyx/python/nyxus/nyxus.py @@ -4,15 +4,14 @@ import pandas as pd from typing import Optional, List -if sys.version_info[0]==3 and sys.version_info[1]>=8: - # since add_dll_dir is added in Python3.8 - # Assuming this is OK since 3.7 has reached EOL and - # this added search_path is needed on conda distribution only which - # currently builds for 3.8+ - import pyarrow as pa - if os.sys.platform == "win32": - for lib_dir in pa.get_library_dirs(): +import pyarrow as pa +if os.sys.platform == "win32": + for lib_dir in pa.get_library_dirs(): + if sys.version_info[0]==3 and sys.version_info[1]>=8: + # since add_dll_dir is added in Python3.8 os.add_dll_directory(lib_dir) + else: + os.environ['PATH'] = lib_dir + os.pathsep + os.environ['PATH'] from .backend import ( initialize_environment,