Fix build, tests, and clean up int types

.github/workflows/cmake.yml

@@ -15,31 +15,37 @@ jobs:
     # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac.
     # You can convert this to a matrix build if you need cross-platform coverage.
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
-    runs-on: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os}}
+    name: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, windows-2022, macos-latest]
 
     steps:
     - uses: actions/checkout@v3
 
     - uses: actions/setup-python@v4
+      name: Install Python 🐍
       with:
         python-version: '3.9'
         cache: 'pip' # caching pip dependencies
 
-    - name: Install Python deps
-      run: pip install numpy scipy cmake
+    - name: Install Python dependencies 🍳
+      run: pip install -r requirements.txt
 
-    - name: Configure CMake
+    - name: Configure CMake 👹
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} 
 
-    - name: Build
+    - name: Build 👷
       # Build your program with the given configuration
-      run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
+      run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j2
 
-    - name: Test
+    - name: Test 🧪
       working-directory: ${{github.workspace}}/build
       # Execute tests defined by the CMake configuration.  
       # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
-      run: ctest -C ${{env.BUILD_TYPE}}
+      run: ctest -C ${{env.BUILD_TYPE}} -V
 

CMakeLists.txt

@@ -5,6 +5,9 @@ project(NumpyEigen VERSION 0.1.0)
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 include(numpyeigen)
 
+# enable C++14
+set(CMAKE_CXX_STANDARD 14)
+
 # Configuration options
 set(NPE_PYTHON_EXECUTABLE    "" CACHE STRING "Path to the Python interpreter")
 set(NPE_PYTHON_VERSION       "" CACHE STRING "Request a specific version of Python")

README.md

@@ -1,4 +1,4 @@
-# NumpyEigen - Fast zero-overhead bindings between NumPy and Eigen [![Build Status](https://travis-ci.com/fwilliams/numpyeigen.svg?branch=master)](https://travis-ci.com/fwilliams/numpyeigen) [![Build status](https://ci.appveyor.com/api/projects/status/hsavlpy5gm767rj5?svg=true)](https://ci.appveyor.com/project/fwilliams/numpyeigen)
+# NumpyEigen - Fast zero-overhead bindings between NumPy and Eigen [![Build Status](https://travis-ci.com/fwilliams/numpyeigen.svg?branch=master)](https://travis-ci.com/fwilliams/numpyeigen) [![Build status](https://ci.appveyor.com/api/projects/status/hsavlpy5gm767rj5?svg=true)](https://ci.appveyor.com/project/fwilliams/numpyeigen)
 
 **NumpyEigen** makes it easy to transparently convert [NumPy](http://www.numpy.org/) 
 dense arrays and [SciPy](https://docs.scipy.org/doc/scipy/reference/sparse.html) sparse 
@@ -109,8 +109,20 @@ make
 make test
 ```
 
-There's an `tests/environment.yml` file which can be used to generate conda environment with a
-ll the right dependencies by running:
+### Windows ❄️
+
+Open a Visual Studio Command Prompt and run the following commands:
+
+```
+mkdir build
+cd build
+cmake ..
+cmake --build . --config Release
+ctest -C Release
+```
+
+There's an `tests/environment.yml` file which can be used to generate conda environment with 
+all the right dependencies by running:
 ```
 conda env create -f environment.yml
 ```

cmake/numpyeigen.cmake

@@ -228,10 +228,13 @@ function(npe_add_module target_name)
     if (npe_add_module_DEBUG_TRACE)
       set(debug_trace_arg "--debug-trace")
     endif()
+
+    # This doesn't seem to be necessary on github ci machines but needed it for local windows build
+    string(REPLACE " " "\\ " CMAKE_CXX_COMPILER_ESCAPED ${CMAKE_CXX_COMPILER})
 
     add_custom_command(OUTPUT ${bound_function_source_filename}
       DEPENDS ${binding_source} ${NPE_SRC_DIR}/codegen_function.py ${NPE_SRC_DIR}/codegen_module.py
-      COMMAND ${PYTHON_EXECUTABLE} ${NPE_SRC_DIR}/codegen_function.py ${binding_source} ${CMAKE_CXX_COMPILER} -o ${bound_function_source_filename} ${debug_trace_arg} --c-preprocessor-args ${C_PREPROCESSOR_CMD_FLAGS}
+      COMMAND ${PYTHON_EXECUTABLE} ${NPE_SRC_DIR}/codegen_function.py ${binding_source} ${CMAKE_CXX_COMPILER_ESCAPED} -o ${bound_function_source_filename} ${debug_trace_arg} --c-preprocessor-args ${C_PREPROCESSOR_CMD_FLAGS}
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
     list(APPEND function_sources "${bound_function_source_filename}")

requirements.txt

@@ -0,0 +1,3 @@
+numpy
+scipy
+cmake>=3.16

src/codegen_function.py

@@ -10,6 +10,12 @@
 import sys
 import tempfile
 
+# Alec: It's not clear what the three values are or wether it's important that
+# they're unique
+#
+# [0] populates cpp_type
+# [1] is char_* in npe_type_defs.h, as in char_int32
+# [2] is used to populate NUMPY_SCALAR_TYPES, pretty_group_types
 """
 Global constants used by NumpyEigen
 """
@@ -20,14 +26,14 @@
     'dense_longdouble': ('npy_longdouble', 'longdouble', 'float128'),
     'dense_byte': ('npy_byte', 'byte', 'int8'),
     'dense_short': ('npy_short', 'short', 'int16'),
-    'dense_int': ('npy_int', 'int', 'int32'),
-    'dense_long': ('npy_long', 'long', 'int64'),
-    'dense_longlong': ('npy_longlong', 'longlong', 'int128'),
+    'dense_int32': ('npy_int32', 'int32', 'int32'),
+    'dense_int64': ('npy_int64', 'int64', 'int64'),
+    'dense_int128': ('npy_int128', 'int128', 'int128'),
     'dense_ubyte': ('npy_ubyte', 'ubyte', 'uint8'),
     'dense_ushort': ('npy_ushort', 'ushort', 'uint16'),
-    'dense_uint': ('npy_int', 'uint', 'uint32'),
-    'dense_ulong': ('npy_ulong', 'ulong', 'uint64'),
-    'dense_ulonglong': ('npy_ulonglong', 'ulonglong', 'uint128'),
+    'dense_uint32': ('npy_uint32', 'uint32', 'uint32'),
+    'dense_uint64': ('npy_uint64', 'uint64', 'uint64'),
+    'dense_uint128': ('npy_uint128', 'uint128', 'uint128'),
     'dense_c64': ('npy_complex64', 'c64', 'complex64'),
     'dense_c128': ('npy_complex128', 'c128', 'complex128'),
     'dense_c256': ('npy_complex256', 'c256', 'complex256'),
@@ -39,14 +45,14 @@
     'sparse_longdouble': ('npy_longdouble', 'longdouble', 'float128'),
     'sparse_byte': ('npy_byte', 'byte', 'int8'),
     'sparse_short': ('npy_short', 'short', 'int16'),
-    'sparse_int': ('npy_int', 'int', 'int32'),
-    'sparse_long': ('npy_long', 'long', 'int64'),
-    'sparse_longlong': ('npy_longlong', 'longlong', 'int128'),
+    'sparse_int32': ('npy_int32', 'int32', 'int32'),
+    'sparse_int64': ('npy_int64', 'int64', 'int64'),
+    'sparse_int128': ('npy_int128', 'int128', 'int128'),
     'sparse_ubyte': ('npy_ubyte', 'ubyte', 'uint8'),
     'sparse_ushort': ('npy_ushort', 'ushort', 'uint16'),
-    'sparse_uint': ('npy_uint', 'uint', 'uint32'),
-    'sparse_ulong': ('npy_ulong', 'ulong', 'uint64'),
-    'sparse_ulonglong': ('npy_ulonglong', 'ulonglong', 'uint128'),
+    'sparse_uint32': ('npy_uint32', 'uint32', 'uint32'),
+    'sparse_uint64': ('npy_uint64', 'uint64', 'uint64'),
+    'sparse_uint128': ('npy_uint128', 'uint128', 'uint128'),
     'sparse_c64': ('npy_complex64', 'c64', 'complex64'),
     'sparse_c128': ('npy_complex128', 'c128', 'complex128'),
     'sparse_c256': ('npy_complex256', 'c256', 'complex256'),
@@ -134,9 +140,28 @@ def run_cpp(input_str):
         if platform.system() == 'Windows':
             cmd = [' '.join(cmd)]
 
-        p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable=cpp_path)
+
+        # Is quoting/escaping really _only_ necessary on Windows, or is it the
+        # only place where a space ends up in the compiler's path?
+        if platform.system() == 'Windows':
+            quoted_cpp_path = f'"{cpp_path}"'
+            quoted_cpp_path = quoted_cpp_path.replace("\\ ", " ")
+            # prepend with cpp_path
+            cmd = [quoted_cpp_path] + cmd
+            # join with spaces into a string
+            cmd = ' '.join(cmd)
+            # This was a mess on a local windows machine. executable= gets very
+            # confused when there's a space in the path.
+            p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        else:
+            p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, executable=cpp_path)
+
+
         cpp_output, cpp_err = p.communicate()
 
+
+
+
         cpp_err = cpp_err.decode("utf-8")
         cpp_err = re.sub(r'(Microsoft \(R\)).+', '', cpp_err)
         cpp_err = re.sub(r'(Copyright \(C\)).+', '', cpp_err)
@@ -787,6 +812,12 @@ def cast_arg(var):
         else:
             raise AssertionError("This should never happen!")
 
+    def cast_arg_data(var):
+        res = cast_arg(var)
+        if var.is_sparse:
+           res += ".data()"
+        return res
+
     def type_name_var(var_name):
         return PRIVATE_ID_PREFIX + var_name + "_type_s"
 
@@ -876,8 +907,12 @@ def write_function_switch_header(fun):
 
         # Declare variables used to determine the type at runtime
         for arg in fun.array_arguments:
-            out_file.write("const char %s = %s::transform_typechar(%s.dtype().type());\n" %
-                           (type_name_var(arg.name), PRIVATE_NAMESPACE, cast_arg(arg)))
+            #out_file.write("const char %s_old = %s.dtype().type();\n" %
+            #               (type_name_var(arg.name), cast_arg(arg)))
+            out_file.write("const char %s = %s::get_type_char(%s);\n" %
+						   (type_name_var(arg.name), PRIVATE_NAMESPACE, cast_arg_data(arg)))
+            #out_file.write("printf(\"%s_old: %%c\\n\", %s_old);\n" % (type_name_var(arg.name),type_name_var(arg.name)))
+            #out_file.write("printf(\"%s: %%c\\n\", %s);\n" % (type_name_var(arg.name),type_name_var(arg.name)))
             out_file.write("ssize_t %s_shape_0 = 0;\n" % arg.name)
             out_file.write("ssize_t %s_shape_1 = 0;\n" % arg.name)
             out_file.write("if (%s.ndim() == 1) {\n" % cast_arg(arg))
@@ -937,7 +972,7 @@ def write_function_switch_header(fun):
             for i in range(len(group_types)):
                 type_name = group_types[i]
                 out_str += type_name_var(first_non_nullable.name) + "!= " + \
-                           PRIVATE_NAMESPACE + "::transform_typechar( " + type_char_for_numpy_type(type_name) + ")"
+                           type_char_for_numpy_type(type_name)
                 next_token = " && " if i < len(group_types) - 1 else ") {\n"
                 out_str += next_token
 
@@ -1032,8 +1067,8 @@ def write_function_switch_body(fun):
                         break
                     repr_var = fun.argument_groups[group_id].arguments[0]
                     typename = combo[group_id][0] + combo[group_id][1]
-                    out_str += type_id_var(repr_var.name) + " == " + PRIVATE_NAMESPACE + "::transform_typeid(" + \
-                               PRIVATE_NAMESPACE + "::" + TYPE_ID_ENUM + "::" + typename + ")"
+                    out_str += type_id_var(repr_var.name) + " == " + \
+                               PRIVATE_NAMESPACE + "::" + TYPE_ID_ENUM + "::" + typename
                     next_token = " && " if group_id < len(combo) - 1 else ")"
                     out_str += next_token
 

src/function_parser.py

@@ -23,10 +23,12 @@ class TermColors:
     'dense_i16': ('std::int16_t', 'i16', 'int16'),
     'dense_i32': ('std::int32_t', 'i32', 'int32'),
     'dense_i64': ('std::int64_t', 'i64', 'int64'),
+    'dense_i128': ('__int128', 'i128', 'int128'),
     'dense_u8': ('std::uint8_t', 'u8', 'uint8'),
     'dense_u16': ('std::uint16_t', 'u16', 'uint16'),
     'dense_u32': ('std::uint32_t', 'u32', 'uint32'),
     'dense_u64': ('std::uint64_t', 'u64', 'uint64'),
+    'dense_u128': ('unsigned __int128', 'u128', 'uint128'),
     'dense_c64': ('std::complex<float>', 'c64', 'complex64'),
     'dense_c128': ('std::complex<double>', 'c128', 'complex128'),
     'dense_c256': ('std::complex<__float128>', 'c256', 'complex256'),
@@ -39,10 +41,12 @@ class TermColors:
     'sparse_i16': ('std::int16_t', 'i16', 'int16'),
     'sparse_i32': ('std::int32_t', 'i32', 'int32'),
     'sparse_i64': ('std::int64_t', 'i64', 'int64'),
+    'sparse_i128': ('__int128', 'i128', 'int128'),
     'sparse_u8': ('std::uint8_t', 'u8', 'uint8'),
     'sparse_u16': ('std::uint16_t', 'u16', 'uint16'),
     'sparse_u32': ('std::uint32_t', 'u32', 'uint32'),
     'sparse_u64': ('std::uint64_t', 'u64', 'uint64'),
+    'sparse_u128': ('unsigned __int128', 'u128', 'uint128'),
     'sparse_c64': ('std::complex<float>', 'c64', 'complex64'),
     'sparse_c128': ('std::complex<double>', 'c128', 'complex128'),
     'sparse_c256': ('std::complex<__float128>', 'c256', 'complex256')}
@@ -559,4 +563,4 @@ def validate_frontend_output(self):
                 if len(self.input_type_groups[group_idx]) == 0:
                     raise SemanticError("Input Variable %s (line %d) was declared with type %s but was "
                                         "unmatched with a numpy type." % (var_name, var_meta.line_number, matches_name))
-                self.input_variable_meta[var_name].is_sparse = self.is_sparse_type(self.input_type_groups[group_idx][0])
+                self.input_variable_meta[var_name].is_sparse = self.is_sparse_type(self.input_type_groups[group_idx][0])

src/npe.h

@@ -35,31 +35,27 @@ namespace npe {
 
     typedef Eigen::Matrix<npy_byte, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_byte;
     typedef Eigen::Matrix<npy_short, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_short;
-    typedef Eigen::Matrix<npy_int, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_int;
-    typedef Eigen::Matrix<npy_long, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_long;
-    typedef Eigen::Matrix<npy_longlong, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_longlong;
+    typedef Eigen::Matrix<npy_int32, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_int32;
+    typedef Eigen::Matrix<npy_int64, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_int64;
 
     typedef Eigen::Matrix<npy_ubyte, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_ubyte;
     typedef Eigen::Matrix<npy_ushort, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_ushort;
     typedef Eigen::Matrix<npy_uint, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_uint;
-    typedef Eigen::Matrix<npy_ulong, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_ulong;
-    typedef Eigen::Matrix<npy_ulonglong, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_ulonglong;
+    typedef Eigen::Matrix<npy_uint64, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_dense_uint64;
 
 
     typedef Eigen::Matrix<npy_double, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_double;
     typedef Eigen::Matrix<npy_float, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_float;
 
     typedef Eigen::Matrix<npy_byte, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_byte;
     typedef Eigen::Matrix<npy_short, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_short;
-    typedef Eigen::Matrix<npy_int, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_int;
-    typedef Eigen::Matrix<npy_long, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_long;
-    typedef Eigen::Matrix<npy_longlong, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_longlong;
+    typedef Eigen::Matrix<npy_int32, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_int32;
+    typedef Eigen::Matrix<npy_int64, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_int64;
 
     typedef Eigen::Matrix<npy_ubyte, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_ubyte;
     typedef Eigen::Matrix<npy_ushort, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_ushort;
     typedef Eigen::Matrix<npy_uint, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_uint;
-    typedef Eigen::Matrix<npy_ulong, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_ulong;
-    typedef Eigen::Matrix<npy_ulonglong, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_ulonglong;
+    typedef Eigen::Matrix<npy_uint64, Eigen::Dynamic, Eigen::Dynamic> npe_Matrix_sparse_uint64;
 
 
 
@@ -68,30 +64,26 @@ namespace npe {
 
     typedef Eigen::Map<npe_Matrix_dense_byte, Eigen::Aligned> sparse_byte;
     typedef Eigen::Map<npe_Matrix_dense_short, Eigen::Aligned> sparse_short;
-    typedef Eigen::Map<npe_Matrix_dense_int, Eigen::Aligned> sparse_int;
-    typedef Eigen::Map<npe_Matrix_dense_long, Eigen::Aligned> sparse_long;
-    typedef Eigen::Map<npe_Matrix_dense_longlong, Eigen::Aligned> sparse_longlong;
+    typedef Eigen::Map<npe_Matrix_dense_int32, Eigen::Aligned> sparse_int32;
+    typedef Eigen::Map<npe_Matrix_dense_int64, Eigen::Aligned> sparse_int64;
 
     typedef Eigen::Map<npe_Matrix_dense_ubyte, Eigen::Aligned> sparse_ubyte;
     typedef Eigen::Map<npe_Matrix_dense_ushort, Eigen::Aligned> sparse_ushort;
     typedef Eigen::Map<npe_Matrix_dense_uint, Eigen::Aligned> sparse_uint;
-    typedef Eigen::Map<npe_Matrix_dense_ulong, Eigen::Aligned> sparse_ulong;
-    typedef Eigen::Map<npe_Matrix_dense_ulonglong, Eigen::Aligned> sparse_ulonglong;
+    typedef Eigen::Map<npe_Matrix_dense_uint64, Eigen::Aligned> sparse_uint64;
 
 
 
     typedef Eigen::Map<npe_Matrix_sparse_double, Eigen::Aligned> dense_double;
     typedef Eigen::Map<npe_Matrix_sparse_float, Eigen::Aligned> dense_float;
     typedef Eigen::Map<npe_Matrix_sparse_byte, Eigen::Aligned> dense_byte;
     typedef Eigen::Map<npe_Matrix_sparse_short, Eigen::Aligned> dense_short;
-    typedef Eigen::Map<npe_Matrix_sparse_int, Eigen::Aligned> dense_int;
-    typedef Eigen::Map<npe_Matrix_sparse_long, Eigen::Aligned> dense_long;
-    typedef Eigen::Map<npe_Matrix_sparse_long, Eigen::Aligned> dense_longlong;
+    typedef Eigen::Map<npe_Matrix_sparse_int32, Eigen::Aligned> dense_int32;
+    typedef Eigen::Map<npe_Matrix_sparse_int64, Eigen::Aligned> dense_int64;
     typedef Eigen::Map<npe_Matrix_sparse_ubyte, Eigen::Aligned> dense_ubyte;
     typedef Eigen::Map<npe_Matrix_sparse_ushort, Eigen::Aligned> dense_ushort;
     typedef Eigen::Map<npe_Matrix_sparse_uint, Eigen::Aligned> dense_uint;
-    typedef Eigen::Map<npe_Matrix_sparse_ulong, Eigen::Aligned> dense_ulong;
-    typedef Eigen::Map<npe_Matrix_sparse_ulonglong, Eigen::Aligned> dense_ulonglong;
+    typedef Eigen::Map<npe_Matrix_sparse_uint64, Eigen::Aligned> dense_uint64;
   }
 
 }

src/npe_dtype.h

@@ -16,13 +16,11 @@ enum DtypeType {
   type_i16     = 'h',
   type_i32     = 'i',
   type_i64     = 'l',
-  type_i128    = 'q',
 
   type_u8      = 'B',
   type_u16     = 'H',
   type_u32     = 'I',
   type_u64     = 'L',
-  type_u128    = 'Q',
 
   type_c64     = 'F',
   type_c128    = 'D',