Merge branch 'branch-0.40' into test-tag-user-callback-discard-return

.pre-commit-config.yaml

@@ -42,7 +42,7 @@ repos:
                 files: \.(h|cpp)$
                 # exclude: path/to/myfile.h
       - repo: https://github.com/rapidsai/pre-commit-hooks
-        rev: v0.3.1
+        rev: v0.4.0
         hooks:
             - id: verify-alpha-spec
               args:

README.md

@@ -6,16 +6,27 @@ UCXX is an object-oriented C++ interface for UCX, with native support for Python
 
 ### Environment setup
 
-Before starting it is necessary to have the necessary dependencies installed. The simplest way to get started is to installed [Miniconda](https://docs.conda.io/en/latest/miniconda.html) and then to create and activate an environment with the provided development file:
+Before starting it is necessary to have the necessary dependencies installed. The simplest way to get started is to install [Miniforge](https://github.com/conda-forge/miniforge) and then to create and activate an environment with the provided development file, for CUDA 11.x:
 
 ```
 $ conda env create -n ucxx -f conda/environments/all_cuda-118_arch-x86_64.yaml
+```
+
+Or for CUDA 12.x:
+
+```
+$ conda env create -n ucxx -f conda/environments/all_cuda-125_arch-x86_64.yaml
+```
+
+And then activate the newly created environment:
+
+```
 $ conda activate ucxx
 ```
 
 #### Faster conda dependency resolution
 
-The procedure aforementioned should complete without issues, but it may be slower than necessary. One alternative to speed up dependency resolution is to install [mamba](https://mamba.readthedocs.io/en/latest/) before creating the new environment. After installing Miniconda, mamba can be installed with:
+The procedure aforementioned should complete without issues, but it may be slower than necessary. One alternative to speed up dependency resolution is to install [mamba](https://mamba.readthedocs.io/en/latest/) before creating the new environment. After installing Miniforge, mamba can be installed with:
 
 ```
 $ conda install -c conda-forge mamba

ci/release/update-version.sh

@@ -55,7 +55,7 @@ for FILE in dependencies.yaml conda/environments/*.yaml; do
     sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
   done
   for DEP in "${UCXX_DEPENDENCIES[@]}"; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/,>=0.0.0a0/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
   done
 done
 for FILE in python/*/pyproject.toml; do

ci/test_common.sh

@@ -173,7 +173,23 @@ install_distributed_dev_mode() {
   # to run non-public API tests in CI.
 
   rapids-logger "Install Distributed in developer mode"
-  git clone https://github.com/dask/distributed /tmp/distributed -b 2024.1.1
+  MAX_ATTEMPTS=5
+  for attempt in $(seq 1 $MAX_ATTEMPTS); do
+    rm -rf /tmp/distributed
+
+    if git clone https://github.com/dask/distributed /tmp/distributed -b 2024.1.1; then
+      break
+    else
+
+      if [ $attempt -eq $MAX_ATTEMPTS ]; then
+        rapids-logger "Maximum number of attempts to clone Distributed failed."
+        exit 1
+      fi
+
+      sleep 1
+    fi
+  done
+
   pip install -e /tmp/distributed
   # `pip install -e` removes files under `distributed` but not the directory, later
   # causing failures to import modules.

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -26,15 +26,15 @@ dependencies:
 - libtool
 - ninja
 - numba>=0.57.1
-- numpy>=1.23,<2.0a0
+- numpy>=1.23,<3.0a0
 - pip
 - pkg-config
 - pre-commit
 - pynvml>=11.4.1
 - pytest-asyncio
 - pytest-rerunfailures
 - pytest==7.*
-- python>=3.9,<3.12
+- python>=3.10,<3.13
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
 - rapids-dask-dependency==24.10.*,>=0.0.0a0
 - rmm==24.10.*,>=0.0.0a0

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -26,15 +26,15 @@ dependencies:
 - libtool
 - ninja
 - numba>=0.57.1
-- numpy>=1.23,<2.0a0
+- numpy>=1.23,<3.0a0
 - pip
 - pkg-config
 - pre-commit
 - pynvml>=11.4.1
 - pytest-asyncio
 - pytest-rerunfailures
 - pytest==7.*
-- python>=3.9,<3.12
+- python>=3.10,<3.13
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
 - rapids-dask-dependency==24.10.*,>=0.0.0a0
 - rmm==24.10.*,>=0.0.0a0

conda/recipes/ucxx/conda_build_config.yaml

@@ -20,9 +20,9 @@ cmake:
   - ">=3.26.4"
 
 python:
-  - 3.9
   - 3.10
   - 3.11
+  - 3.12
 
 ucx:
   - "==1.15.*"

conda/recipes/ucxx/meta.yaml

@@ -239,7 +239,7 @@ outputs:
         - ucx >=1.15.0,<1.18.0
         - {{ pin_subpackage('libucxx', exact=True) }}
         - {{ pin_compatible('rmm', max_pin='x.x') }}
-        - numpy>=1.23,<2.0a0
+        - numpy>=1.23,<3.0a0
         - pynvml >=11.4.1
       run_constrained:
         - cupy >=9.5.0

cpp/python/src/future.cpp

@@ -113,41 +113,13 @@ PyObject* create_python_future()
   return result;
 }
 
-static PyCFunction get_future_method(const char* method_name)
-{
-  PyCFunction result = NULL;
-
-  PyGILState_STATE state = PyGILState_Ensure();
-
-  PyObject* future_object = get_asyncio_future_object();
-  if (PyErr_Occurred()) {
-    ucxx_trace_req("ucxx::python::%s, error getting asyncio.Future method object", __func__);
-    PyErr_Print();
-  }
-  PyMethodDef* m = reinterpret_cast<PyTypeObject*>(future_object)->tp_methods;
-
-  for (; m != NULL; ++m) {
-    if (m->ml_name && !strcmp(m->ml_name, method_name)) {
-      result = m->ml_meth;
-      break;
-    }
-  }
-
-  if (!result)
-    PyErr_Format(PyExc_RuntimeError, "Unable to load function pointer for `Future.set_result`.");
-
-  PyGILState_Release(state);
-  return result;
-}
-
 PyObject* future_set_result(PyObject* future, PyObject* value)
 {
   PyObject* result = NULL;
 
   PyGILState_STATE state = PyGILState_Ensure();
 
-  PyCFunction f = get_future_method("set_result");
-  result        = f(future, value);
+  result = PyObject_CallMethodOneArg(future, set_result_str, value);
   if (PyErr_Occurred()) {
     ucxx_trace_req("ucxx::python::%s, error calling `set_result()` from `asyncio.Future` object",
                    __func__);
@@ -165,7 +137,6 @@ PyObject* future_set_exception(PyObject* future, PyObject* exception, const char
   PyObject* message_object   = NULL;
   PyObject* message_tuple    = NULL;
   PyObject* formed_exception = NULL;
-  PyCFunction f              = NULL;
 
   PyGILState_STATE state = PyGILState_Ensure();
 
@@ -176,9 +147,8 @@ PyObject* future_set_exception(PyObject* future, PyObject* exception, const char
   formed_exception = PyObject_Call(exception, message_tuple, NULL);
   if (formed_exception == NULL) goto err;
 
-  f = get_future_method("set_exception");
+  result = PyObject_CallMethodOneArg(future, set_exception_str, formed_exception);
 
-  result = f(future, formed_exception);
   goto finish;
 
 err:

dependencies.yaml

@@ -263,10 +263,6 @@ dependencies:
     specific:
       - output_types: conda
         matrices:
-          - matrix:
-              py: "3.9"
-            packages:
-              - python=3.9
           - matrix:
               py: "3.10"
             packages:
@@ -275,14 +271,18 @@ dependencies:
               py: "3.11"
             packages:
               - python=3.11
+          - matrix:
+              py: "3.12"
+            packages:
+              - python=3.12
           - matrix:
             packages:
-              - python>=3.9,<3.12
+              - python>=3.10,<3.13
   run_python_ucxx:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - &numpy numpy>=1.23,<2.0a0
+          - &numpy numpy>=1.23,<3.0a0
           - pynvml>=11.4.1
   run_python_distributed_ucxx:
     common:

python/distributed-ucxx/pyproject.toml

@@ -14,7 +14,7 @@ authors = [
     { name = "NVIDIA Corporation" },
 ]
 license = { text = "BSD-3-Clause" }
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "numba>=0.57.1",
     "rapids-dask-dependency==24.10.*,>=0.0.0a0",
@@ -24,9 +24,9 @@ classifiers = [
     "Intended Audience :: Developers",
     "License :: OSI Approved :: Apache Software License",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 
 [project.entry-points."distributed.comm.backends"]
@@ -41,7 +41,7 @@ docs = [
 test = [
     "cudf==24.10.*,>=0.0.0a0",
     "cupy-cuda11x>=12.0.0",
-    "numpy>=1.23,<2.0a0",
+    "numpy>=1.23,<3.0a0",
     "pytest-rerunfailures",
     "pytest==7.*",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

python/ucxx/pyproject.toml

@@ -17,10 +17,10 @@ authors = [
     { name = "NVIDIA Corporation" },
 ]
 license = { text = "BSD-3-Clause" }
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "libucxx==0.40.*,>=0.0.0a0",
-    "numpy>=1.23,<2.0a0",
+    "numpy>=1.23,<3.0a0",
     "pynvml>=11.4.1",
     "rmm==24.10.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
@@ -73,7 +73,7 @@ ninja.make-fallback = true
 sdist.exclude = ["*tests*"]
 sdist.reproducible = true
 wheel.packages = ["ucxx"]
-wheel.exclude = ["*.pyx", "*.pxd", "CMakeLists.txt"]
+wheel.exclude = ["*.pyx", "CMakeLists.txt"]
 
 [tool.scikit-build.metadata.version]
 provider = "scikit_build_core.metadata.regex"

python/ucxx/ucxx/__init__.py

@@ -103,10 +103,13 @@ def _is_mig_device(handle):
 
 from ._version import __git_commit__, __version__
 
+__ucx_min_version__ = "1.15.0"
 __ucx_version__ = "%d.%d.%d" % get_ucx_version()
 
-if get_ucx_version() < (1, 11, 1):
+if get_ucx_version() < tuple(int(i) for i in __ucx_min_version__.split(".")):
     raise ImportError(
         f"Support for UCX {__ucx_version__} has ended. Please upgrade to "
-        "1.11.1 or newer."
+        f"{__ucx_min_version__} or newer. If you believe the wrong version "
+        "is being loaded, please check the path from where UCX is loaded "
+        "by rerunning with the environment variable `UCX_LOG_LEVEL=debug`."
     )