Numpy 2.1+ support (#11)

* pin to numpy < 2.1 * no 3.13 yet * set numpy 2.0 pin * numpy1 * restructure defines and undef NPY_API_SYMBOL_ATTRIBUTE * undo pin * parameterize numpy * doh * exclude 2.1 for python 3.9 * use _import_array as before for numpy 1.x * cleanup and fix osx flat_namespace thing * put back NPY_API_SYMBOL_ATTRIBUTE * set min CMake version to 3.13 * bump ver to 1.6.2
PDAL · Sep 24, 2024 · b00cc4e · b00cc4e
1 parent 9b8199f
commit b00cc4e
Show file tree

Hide file tree

Showing 10 changed files with 57 additions and 45 deletions.
diff --git a/.github/environment.yml b/.github/environment.yml
@@ -8,3 +8,7 @@ dependencies:
   - compilers
   - python
   - pip
+  - scikit-build-core
+  - pybind11
+  - cmake
+  - ninja
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -17,14 +17,18 @@ defaults:
 
 jobs:
   build:
-    name: Build ${{ matrix.os }} py${{ matrix.python-version }}
+    name: ${{ matrix.os }} py${{ matrix.python-version }} numpy${{ matrix.numpy-version }}
     runs-on: ${{ matrix.os }}
 
     strategy:
       fail-fast: true
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
         python-version: ['3.9', '3.10', '3.11', '3.12']
+        numpy-version: ['1.26', '2.0', '2.1']
+        exclude:
+          - python-version: 3.9
+            numpy-version: 2.1
 
     steps:
     - name: Check out
@@ -40,6 +44,10 @@ jobs:
         auto-update-conda: true
         environment-file: .github/environment.yml
 
+    - name: Install Numpy ${{ matrix.numpy-version }}
+      shell: bash -l {0}
+      run: |
+        mamba install numpy=${{ matrix.numpy-version }}
 
     - name: Install
       shell: bash -l {0}

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,4 @@
-cmake_minimum_required(VERSION 3.11.0)
-project(pdal-python-plugins)
+cmake_minimum_required(VERSION 3.13.0)
 project(pdal-python-plugins VERSION ${SKBUILD_PROJECT_VERSION}
                             DESCRIPTION "PDAL Python Plugins"
                             HOMEPAGE_URL "https://github.com/PDAL/python-plugins")

diff --git a/pyproject.toml b/pyproject.toml
@@ -29,7 +29,7 @@ dependencies = [
   "numpy >= 1.22"
 ]
 
-version="1.6.1"
+version="1.6.2"
 
 [project.optional-dependencies]
 test = [ ]

diff --git a/src/pdal/io/NumpyReader.cpp b/src/pdal/io/NumpyReader.cpp
@@ -43,7 +43,7 @@
 #include <pdal/util/Algorithm.hpp>
 #include <pdal/util/Extractor.hpp>
 
-#include "../plang/Environment.hpp"
+
 
 #if NPY_ABI_VERSION < 0x02000000
   #define PyDataType_FIELDS(descr) ((descr)->fields)
@@ -138,10 +138,13 @@ void NumpyReader::setArray(PyObject* array)
     Py_XINCREF(m_array);
 }
 
+
 PyArrayObject* load_npy_file(std::string const& filename)
 {
 
     PyObject *py_filename =  PyUnicode_FromString(filename.c_str());
+    if (!py_filename)
+        throw pdal::pdal_error(plang::getTraceback());
     PyObject *numpy_module = PyImport_ImportModule("numpy");
     if (!numpy_module)
         throw pdal::pdal_error(plang::getTraceback());
@@ -165,7 +168,11 @@ PyArrayObject* load_npy_file(std::string const& filename)
     if (!array)
         throw pdal::pdal_error(plang::getTraceback());
 
-    return reinterpret_cast<PyArrayObject*>(array);
+
+    PyArrayObject* nparray = reinterpret_cast<PyArrayObject*>(array);
+    if (!PyArray_Check(array))
+        throw pdal_error("Numpy file did not return an array!");
+    return nparray;
 }
 
 PyArrayObject* load_npy_script(std::string const& source,
@@ -196,6 +203,9 @@ PyArrayObject* load_npy_script(std::string const& source,
 
     Py_XDECREF(scriptArgs);
 
+    if (!PyArray_Check(array))
+        throw pdal_error("Numpy script did not return an array!");
+
     return reinterpret_cast<PyArrayObject*>(array);
 }
 
@@ -350,32 +360,24 @@ Dimension::Id NumpyReader::registerDim(PointLayoutPtr layout,
     return id;
 }
 
-namespace
-{
 
-
-Dimension::Type getType(PyArray_Descr *dtype, const std::string& name)
+void NumpyReader::createFields(PointLayoutPtr layout)
 {
-    if (!dtype)
-        throw pdal_error("Can't fetch data type for numpy field.");
 
-    Dimension::Type pdalType =
-        plang::Environment::getPDALDataType(dtype->type_num);
-    if (pdalType == Dimension::Type::None)
+    auto getPDALType = [](int type_num, const std::string& name)
     {
-        std::ostringstream oss;
-        oss << "Unable to map dimension '" << name << "' because its "
-            "type '" << dtype->type_num <<"' is not mappable to PDAL";
-        throw pdal_error(oss.str());
-    }
-    return pdalType;
-}
-
-} // unnamed namespace
-
+        Dimension::Type pdalType =
+            plang::Environment::getPDALDataType(type_num);
+        if (pdalType == Dimension::Type::None)
+        {
+            std::ostringstream oss;
+            oss << "Unable to map dimension '" << name << "' because its "
+                "type '" << type_num <<"' is not mappable to PDAL";
+            throw pdal_error(oss.str());
+        }
+        return pdalType;
+    };
 
-void NumpyReader::createFields(PointLayoutPtr layout)
-{
     Dimension::Id id;
     Dimension::Type type;
     int offset;
@@ -388,7 +390,7 @@ void NumpyReader::createFields(PointLayoutPtr layout)
     // Array isn't structured - just a bunch of data.
     if (m_numFields <= 0)
     {
-        type = getType(m_dtype, m_defaultDimension);
+        type = getPDALType(m_dtype->type_num, m_defaultDimension);
         id = registerDim(layout, m_defaultDimension, type);
         m_fields.push_back({id, type, 0});
     }
@@ -415,7 +417,7 @@ void NumpyReader::createFields(PointLayoutPtr layout)
 
             // Get type.
             PyArray_Descr* dt = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0);
-            type = getType(dt, name);
+            type = getPDALType(dt->type_num, name);
 
             char byteorder = dt->byteorder;
             int elsize = (int) PyDataType_ELSIZE(dt);

diff --git a/src/pdal/io/NumpyReader.hpp b/src/pdal/io/NumpyReader.hpp
@@ -37,13 +37,11 @@
 #include <pdal/Reader.hpp>
 #include <pdal/Streamable.hpp>
 
+#include "../plang/Environment.hpp"
 #include "../plang/Invocation.hpp"
 
-#include <Python.h>
-#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION
-#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API
-#define NO_IMPORT_ARRAY
-#include <numpy/arrayobject.h>
+#define NO_IMPORT_ARRAY // Already have it from Environment.hpp
+#include <numpy/ndarrayobject.h>
 
 #include <memory>
 

diff --git a/src/pdal/plang/Environment.cpp b/src/pdal/plang/Environment.cpp
@@ -39,12 +39,8 @@
 #include "Environment.hpp"
 #include "Redirector.hpp"
 
-#define NPY_TARGET_VERSION NPY_1_22_API_VERSION
-#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION
 
-#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API
-
-#include <numpy/arrayobject.h>
+#include <numpy/ndarrayobject.h>
 #include <pdal/util/FileUtils.hpp>
 #include <pdal/util/Utils.hpp>
 
@@ -145,9 +141,12 @@ Environment::Environment()
     // the return.
     auto initNumpy = []()
     {
-// #undef NUMPY_IMPORT_ARRAY_RETVAL
-// #define NUMPY_IMPORT_ARRAY_RETVAL VOID
+
+#if NPY_ABI_VERSION < 0x02000000
         _import_array();
+#else
+        PyArray_ImportNumPyAPI();
+#endif
         return ;
     };
 
@@ -165,7 +164,6 @@ Environment::Environment()
             throw pdal_error("unable to add redirector module!");
     }
 
-
     initNumpy();
     PyImport_ImportModule("redirector");
 

diff --git a/src/pdal/plang/Environment.hpp b/src/pdal/plang/Environment.hpp
@@ -49,6 +49,11 @@
 #include "Script.hpp"
 #include "gil.hpp"
 
+#define PY_ARRAY_UNIQUE_SYMBOL PDAL_NPARRAY_API
+#define NPY_TARGET_VERSION NPY_1_22_API_VERSION
+#define NPY_API_SYMBOL_ATTRIBUTE
+#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION
+
 namespace pdal
 {
 namespace plang

diff --git a/src/pdal/plang/Invocation.cpp b/src/pdal/plang/Invocation.cpp
@@ -36,10 +36,8 @@
 
 #include <pdal/util/Algorithm.hpp>
 
-#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION
 #define NO_IMPORT_ARRAY
-#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API
-#include <numpy/arrayobject.h>
+#include <numpy/ndarrayobject.h>
 
 namespace
 {

diff --git a/src/pdal/test/data/1.2-with-color.npy b/src/pdal/test/data/1.2-with-color.npy