Adding Combination Operators Zip, CombineLatest, WithLatestFrom

.clang-tidy

@@ -20,7 +20,11 @@ Checks: >
   -modernize-macro-to-enum,
   google-readability-todo
 
-#WarningsAsErrors: '*'
+WarningsAsErrors: >
+  *,
+  -clang-diagnostic-unused-command-line-argument,
+  -clang-diagnostic-ignored-optimization-argument
+
 HeaderFilterRegex: '.*\/include\/mrc\/.*'
 AnalyzeTemporaryDtors: false
 FormatStyle: file

.devcontainer/conda/Dockerfile

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:23.04-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin"

.devcontainer/opt/mrc/bin/post-attach-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,6 +28,6 @@ sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
 
-then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml; \
+then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-125_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-125_arch-x86_64.yaml; \
 fi

.devcontainer/opt/mrc/conda/Dockerfile

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:23.04-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin"

.github/copy-pr-bot.yaml

@@ -0,0 +1,4 @@
+# Configuration file for `copy-pr-bot` GitHub App
+# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/
+
+enabled: true

.github/ops-bot.yaml

@@ -5,5 +5,4 @@ auto_merger: true
 branch_checker: true
 label_checker: true
 release_drafter: true
-copy_prs: true
-rerun_tests: true
+forward_merger: true

.github/workflows/ci_pipe.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -61,6 +61,7 @@ env:
   GH_TOKEN: "${{ github.token }}"
   GIT_COMMIT: "${{ github.sha }}"
   MRC_ROOT: "${{ github.workspace }}/mrc"
+  RAPIDS_CONDA_RETRY_MAX: "5"
   WORKSPACE: "${{ github.workspace }}/mrc"
   WORKSPACE_TMP: "${{ github.workspace }}/tmp"
 
@@ -294,7 +295,7 @@ jobs:
         run: ./mrc/ci/scripts/github/benchmark.sh
       - name: post_benchmark
         shell: bash
-        run: ./mrc/ci/scripts/github/benchmark.sh
+        run: ./mrc/ci/scripts/github/post_benchmark.sh
 
 
   package:

.github/workflows/pull_request.yml → .github/workflows/pr.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -43,11 +43,18 @@ permissions:
   statuses: none
 
 jobs:
+  pr-builder:
+    needs:
+      - checks
+      - prepare
+      - ci_pipe
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
   prepare:
     name: Prepare
     runs-on: ubuntu-latest
     container:
-      image: rapidsai/ci:latest
+      image: rapidsai/ci-conda:latest
     steps:
       - name: Get PR Info
         id: get-pr-info
@@ -58,11 +65,22 @@ jobs:
       is_main_branch: ${{ github.ref_name == 'main' }}
       is_dev_branch: ${{ startsWith(github.ref_name, 'branch-') }}
       has_conda_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-build') || false }}
+      has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }}
       pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }}
+
+  checks:
+    needs: [prepare]
+    if: ${{ !fromJSON(needs.prepare.outputs.has_skip_ci_label) && fromJSON(needs.prepare.outputs.is_pr )}}
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      enable_check_generated_files: false
+
   ci_pipe:
     name: CI Pipeline
     needs: [prepare]
     uses: ./.github/workflows/ci_pipe.yml
+    if: ${{ ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }}
     with:
       # Run checks for any PR branch
       run_check: ${{ fromJSON(needs.prepare.outputs.is_pr) }}
@@ -71,9 +89,9 @@ jobs:
       # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       # Build container
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230711
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-241002
       # Test container
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230711
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-241002
       # Info about the PR. Empty for non PR branches. Useful for extracting PR number, title, etc.
       pr_info: ${{ needs.prepare.outputs.pr_info }}
     secrets:

.gitignore

@@ -1,4 +1,5 @@
 /build*/
+.tmp
 *.engine
 .Dockerfile
 .gitignore
@@ -17,6 +18,9 @@ include/mrc/version.hpp
 .vscode/settings.json
 .vscode/tasks.json
 
+# Ignore user-defined clangd settings
+.clangd
+
 # Created by https://www.gitignore.io/api/vim,c++,cmake,python,synology
 
 ### C++ ###

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "morpheus_utils"]
 	path = external/utilities
 	url = https://github.com/nv-morpheus/utilities.git
-	branch = branch-23.07
+	branch = branch-24.10

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+repos:
+      - repo: https://github.com/rapidsai/dependency-file-generator
+        rev: v1.13.11
+        hooks:
+            - id: rapids-dependency-file-generator
+              args: ["--clean"]
+      - repo: https://github.com/PyCQA/isort
+        rev: 5.12.0
+        hooks:
+              - id: isort
+                args: ["--settings-file=./python/setup.cfg"]
+                files: ^python/
+      - repo: https://github.com/PyCQA/flake8
+        rev: 6.1.0
+        hooks:
+              - id: flake8
+                args: ["--config=./python/setup.cfg"]
+                files: ^python/
+      - repo: https://github.com/google/yapf
+        rev: v0.40.2
+        hooks:
+              - id: yapf
+                args: ["--style", "./python/setup.cfg"]
+                files: ^python/
+
+default_language_version:
+      python: python3

CHANGELOG.md

@@ -1,3 +1,111 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# MRC 24.06.00 (03 Jul 2024)
+
+## 🚀 New Features
+
+- Add JSONValues container for holding Python values as JSON objects if possible, and as pybind11::object otherwise ([#455](https://github.com/nv-morpheus/MRC/pull/455)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 🛠️ Improvements
+
+- resolve rapids-dependency-file-generator warning ([#482](https://github.com/nv-morpheus/MRC/pull/482)) [@jameslamb](https://github.com/jameslamb)
+- Downgrade doxygen to match Morpheus ([#469](https://github.com/nv-morpheus/MRC/pull/469)) [@cwharris](https://github.com/cwharris)
+- Consolidate redundant split_string_to_array, split_string_on &amp; split_path methods ([#465](https://github.com/nv-morpheus/MRC/pull/465)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Add pybind11 type caster for JSONValues ([#458](https://github.com/nv-morpheus/MRC/pull/458)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+# MRC 24.03.01 (16 Apr 2024)
+
+## 🐛 Bug Fixes
+
+- Add auto register helpers to AsyncSink and AsyncSource ([#473](https://github.com/nv-morpheus/MRC/pull/473)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+# MRC 24.03.00 (7 Apr 2024)
+
+## 🚨 Breaking Changes
+
+- Update cast_from_pyobject to throw on unsupported types rather than returning null ([#451](https://github.com/nv-morpheus/MRC/pull/451)) [@dagardner-nv](https://github.com/dagardner-nv)
+- RAPIDS 24.02 Upgrade ([#433](https://github.com/nv-morpheus/MRC/pull/433)) [@cwharris](https://github.com/cwharris)
+
+## 🐛 Bug Fixes
+
+- Update CR year ([#460](https://github.com/nv-morpheus/MRC/pull/460)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Removing the INFO log when creating an AsyncioRunnable ([#456](https://github.com/nv-morpheus/MRC/pull/456)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Update cast_from_pyobject to throw on unsupported types rather than returning null ([#451](https://github.com/nv-morpheus/MRC/pull/451)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Adopt updated builds of CI runners ([#442](https://github.com/nv-morpheus/MRC/pull/442)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Update Conda channels to prioritize `conda-forge` over `nvidia` ([#436](https://github.com/nv-morpheus/MRC/pull/436)) [@cwharris](https://github.com/cwharris)
+- Remove redundant copy of libmrc_pymrc.so ([#429](https://github.com/nv-morpheus/MRC/pull/429)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Unifying cmake exports name across all Morpheus repos ([#427](https://github.com/nv-morpheus/MRC/pull/427)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Updating the workspace settings to remove deprecated python options ([#425](https://github.com/nv-morpheus/MRC/pull/425)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Use `dependencies.yaml` to generate environment files ([#416](https://github.com/nv-morpheus/MRC/pull/416)) [@cwharris](https://github.com/cwharris)
+
+## 📖 Documentation
+
+- Update minimum requirements ([#467](https://github.com/nv-morpheus/MRC/pull/467)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 🚀 New Features
+
+- Add maximum simultaneous tasks support to `TaskContainer` ([#464](https://github.com/nv-morpheus/MRC/pull/464)) [@cwharris](https://github.com/cwharris)
+- Add `TestScheduler` to support testing time-based coroutines without waiting for timeouts ([#453](https://github.com/nv-morpheus/MRC/pull/453)) [@cwharris](https://github.com/cwharris)
+- Adding RoundRobinRouter node type for distributing values to downstream nodes ([#449](https://github.com/nv-morpheus/MRC/pull/449)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Add IoScheduler to enable epoll-based Task scheduling ([#448](https://github.com/nv-morpheus/MRC/pull/448)) [@cwharris](https://github.com/cwharris)
+- Update ops-bot.yaml ([#446](https://github.com/nv-morpheus/MRC/pull/446)) [@AyodeAwe](https://github.com/AyodeAwe)
+- RAPIDS 24.02 Upgrade ([#433](https://github.com/nv-morpheus/MRC/pull/433)) [@cwharris](https://github.com/cwharris)
+
+## 🛠️ Improvements
+
+- Update MRC to use CCCL instead of libcudacxx ([#444](https://github.com/nv-morpheus/MRC/pull/444)) [@cwharris](https://github.com/cwharris)
+- Optionally skip the CI pipeline if the PR contains the skip-ci label ([#426](https://github.com/nv-morpheus/MRC/pull/426)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Add flake8, yapf, and isort pre-commit hooks. ([#420](https://github.com/nv-morpheus/MRC/pull/420)) [@cwharris](https://github.com/cwharris)
+
+# MRC 23.11.00 (30 Nov 2023)
+
+## 🐛 Bug Fixes
+
+- Use a traditional semaphore in AsyncioRunnable ([#412](https://github.com/nv-morpheus/MRC/pull/412)) [@cwharris](https://github.com/cwharris)
+- Fix libhwloc &amp; stubgen versions to match dev yaml ([#405](https://github.com/nv-morpheus/MRC/pull/405)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Update boost versions to match version used in dev env ([#404](https://github.com/nv-morpheus/MRC/pull/404)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fix EdgeHolder from incorrectly reporting an active connection ([#402](https://github.com/nv-morpheus/MRC/pull/402)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Safe handling of control plane promises &amp; fix CI ([#391](https://github.com/nv-morpheus/MRC/pull/391)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Revert boost upgrade, and update clang to v16 ([#382](https://github.com/nv-morpheus/MRC/pull/382)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fixing an issue with `update-versions.sh` which always blocked CI ([#377](https://github.com/nv-morpheus/MRC/pull/377)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Add test for  gc being invoked in a thread finalizer ([#365](https://github.com/nv-morpheus/MRC/pull/365)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Adopt patched pybind11 ([#364](https://github.com/nv-morpheus/MRC/pull/364)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 📖 Documentation
+
+- Add missing flags to docker command to mount the working dir and set -cap-add=sys_nice ([#383](https://github.com/nv-morpheus/MRC/pull/383)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Make Quick Start Guide not use `make_node_full` ([#376](https://github.com/nv-morpheus/MRC/pull/376)) [@cwharris](https://github.com/cwharris)
+
+## 🚀 New Features
+
+- Add AsyncioRunnable ([#411](https://github.com/nv-morpheus/MRC/pull/411)) [@cwharris](https://github.com/cwharris)
+- Adding more coroutine components to support async generators and task containers ([#408](https://github.com/nv-morpheus/MRC/pull/408)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Update ObservableProxy::pipe to support any number of operators ([#387](https://github.com/nv-morpheus/MRC/pull/387)) [@cwharris](https://github.com/cwharris)
+- Updates for MRC/Morpheus to build in the same RAPIDS devcontainer environment ([#375](https://github.com/nv-morpheus/MRC/pull/375)) [@cwharris](https://github.com/cwharris)
+
+## 🛠️ Improvements
+
+- Move Pycoro from Morpheus to MRC ([#409](https://github.com/nv-morpheus/MRC/pull/409)) [@cwharris](https://github.com/cwharris)
+- update rapidsai/ci to rapidsai/ci-conda ([#396](https://github.com/nv-morpheus/MRC/pull/396)) [@AyodeAwe](https://github.com/AyodeAwe)
+- Add local CI scripts &amp; rebase docker image ([#394](https://github.com/nv-morpheus/MRC/pull/394)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Use `copy-pr-bot` ([#369](https://github.com/nv-morpheus/MRC/pull/369)) [@ajschmidt8](https://github.com/ajschmidt8)
+- Update Versions for v23.11.00 ([#357](https://github.com/nv-morpheus/MRC/pull/357)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+
 # MRC 23.07.00 (19 Jul 2023)
 
 ## 🚨 Breaking Changes

CMakeLists.txt

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,13 +30,14 @@ option(MRC_BUILD_PYTHON "Enable building the python bindings for MRC" ON)
 option(MRC_BUILD_TESTS "Whether or not to build MRC tests" ON)
 option(MRC_ENABLE_CODECOV "Enable gcov code coverage" OFF)
 option(MRC_ENABLE_DEBUG_INFO "Enable printing debug information" OFF)
+option(MRC_PYTHON_INPLACE_BUILD "Whether or not to copy built python modules back to the source tree for debug purposes." OFF)
 option(MRC_USE_CCACHE "Enable caching compilation results with ccache" OFF)
 option(MRC_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF)
 option(MRC_USE_CONDA "Enables finding dependencies via conda. All dependencies must be installed first in the conda
   environment" ON)
 option(MRC_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
 
-set(MRC_RAPIDS_VERSION "23.06" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
+set(MRC_RAPIDS_VERSION "24.10" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
 
 set(MRC_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MRC_CACHE_DIR)
@@ -78,10 +79,12 @@ morpheus_utils_initialize_package_manager(
 morpheus_utils_initialize_cuda_arch(mrc)
 
 project(mrc
-  VERSION 23.07.00
+  VERSION 24.10.00
   LANGUAGES C CXX
 )
 
+morpheus_utils_initialize_install_prefix(MRC_USE_CONDA)
+
 rapids_cmake_write_version_file(${CMAKE_BINARY_DIR}/autogenerated/include/mrc/version.hpp)
 
 # Delay enabling CUDA until after we have determined our CXX compiler

CONTRIBUTING.md

@@ -85,7 +85,7 @@ cd $MRC_ROOT
 #### Create MRC Conda environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc --file $MRC_ROOT/ci/conda/environments/dev_env.yml
+conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-125_arch-x86_64.yaml
 conda activate mrc
 ```
 #### Build MRC