♻️plit ResourcePool into three classes

Squashed commit of the following: commit 0a53108 Author: Jon Clucas <[email protected]> Date: Fri Jul 19 01:16:36 2024 -0400 :goal_net: Catch and release no-regressors commit 836d100 Author: Jon Clucas <[email protected]> Date: Fri Jul 19 00:50:16 2024 -0400 :art: Remove duplicate imports commit 8d80941 Author: Jon Clucas <[email protected]> Date: Thu Jul 18 21:01:35 2024 -0400 :art: Qualify refs to documented functions. commit 2402a2c Author: Jon Clucas <[email protected]> Date: Thu Jul 18 20:22:13 2024 -0400 :pencil2: A little more docstring cleanup commit 61eeef4 Author: Jon Clucas <[email protected]> Date: Thu Jul 18 20:08:08 2024 -0400 :bug: Fix memoization commit 763af4e Author: Jon Clucas <[email protected]> Date: Thu Jul 18 14:30:35 2024 -0400 :art: More docstring updates for ResourcePool refactor commit 6c96667 Author: Jon Clucas <[email protected]> Date: Thu Jul 18 12:59:32 2024 -0400 :bug: Fix conflicting class name commit f5dd824 Author: Jon Clucas <[email protected]> Date: Thu Jul 18 12:41:45 2024 -0400 :art: Standardize docstring format across changes. commit 75e38e0 Author: Jon Clucas <[email protected]> Date: Thu Jul 18 10:57:21 2024 -0400 :memo: Update CHANGELOG re: #2131 commit 3b00764 Merge: 7b04cc8 a40c449 Author: Jon Clucas <[email protected]> Date: Thu Jul 18 09:47:22 2024 -0400 :twisted_rightwards_arrows: Merge 'origin/develop' into 'engine/(th)re(e)sources' commit 7b04cc8 Author: Jon Clucas <[email protected]> Date: Thu Jul 18 09:42:39 2024 -0400 :white_check_mark: Unlink symlink instead of rmtree commit dd0985f Author: Jon Clucas <[email protected]> Date: Thu Jul 18 09:24:29 2024 -0400 :pencil2: Fix typo (~~"tpyes"~~ → "types") commit 4a05442 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 17:04:43 2024 -0400 :construction_worker: Pre-clone `bids-examples` commit 7571455 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:54:50 2024 -0400 :recycle: Move `_check_null` from method to private function commit 0d848b9 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:44:34 2024 -0400 :recycle: Replace calls to `grab_tiered_dct` with direct config lookup commit a855222 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:24:39 2024 -0400 :coffin: Remove unused function `run_node_blocks` commit 2e97e51 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:23:21 2024 -0400 :recycle: Move `initialize_nipype_wf` method into `ResourcePool` commit 8389d5b Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:21:51 2024 -0400 :recycle: Move `json_outdir_ingress` method into `ResourcePool` commit 84274b9 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:20:19 2024 -0400 :recycle: Move `_config_lookup` method into `ResourcePool` commit cf85273 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:16:48 2024 -0400 :coffin: Remove unused `get_name` method commit c944940 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:13:05 2024 -0400 :recycle: Move `create_func_datasource` method into `ResourcePool` commit ef402e5 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:11:07 2024 -0400 :recycle: Move `_get_pipe_number` back into `ResourcePool` commit 77ffc16 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 16:05:08 2024 -0400 :coffin: Remove unused `wrap_block` function commit 14c3e32 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:44:44 2024 -0400 :coffin: Remove unused `get_entire_rpool` method commit 2fc7244 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:39:26 2024 -0400 :recycle: Move `get_raw_label` back into `ResourcePool` commit b8ca36c Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:38:49 2024 -0400 :coffin: Remove unused `get_strat_info` method commit 839c7cd Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:38:26 2024 -0400 :coffin: Remove unused `set_json_info` method commit 0a0b5a0 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:37:39 2024 -0400 :recycle: Move `get_json_info` back into `ResourcePool` commit 69cb603 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:28:21 2024 -0400 :recycle: Move `copy_resource` method into `StratPool` commit a004ab6 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:25:59 2024 -0400 :coffin: Remove unused `update_resource` method commit 6fb1dc1 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:25:33 2024 -0400 :coffin: Remove unused `get_pipe_idxs` method commit 0f9099c Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:25:03 2024 -0400 :recycle: Split `get_json` across `ResourcePool` and `StratPool` commit f2423a2 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 15:15:37 2024 -0400 :recycle: Move `get_cpac_provenance` and `regressor_dct` into `StratPool` commit 52c38bf Author: Jon Clucas <[email protected]> Date: Wed Jul 17 14:49:30 2024 -0400 :coffin: Remove unused `generate_prov_list` method commit 748b98e Author: Jon Clucas <[email protected]> Date: Wed Jul 17 14:48:56 2024 -0400 :recycle: Move `get_resource_strats_from_prov` back into `ResourcePool` commit 74629ac Author: Jon Clucas <[email protected]> Date: Wed Jul 17 14:44:18 2024 -0400 :coffin: Remove unused `flatten_prov` method commit 48e8b90 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 14:43:34 2024 -0400 :recycle: Move `derivative_xfm` back into `ResourcePool` commit 29b481d Author: Jon Clucas <[email protected]> Date: Wed Jul 17 14:25:12 2024 -0400 :recycle: Move `filtered_movement` property into `StratPool` commit b0b94c9 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 14:23:36 2024 -0400 :recycle: Move `filter_name` method into `StratPool` commit 4bf5f00 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 14:20:36 2024 -0400 :recycle: Move `post_process` method back into `ResourcePool` commit 3613f8c Author: Jon Clucas <[email protected]> Date: Wed Jul 17 14:16:16 2024 -0400 :construction_worker: Livelog pytest commit a9a3c48 Author: Jon Clucas <[email protected]> Date: Wed Jul 17 13:41:27 2024 -0400 :coffin: Remove `_Pool.node_data` method commit d30496c Author: Jon Clucas <[email protected]> Date: Wed Jul 17 13:28:36 2024 -0400 :recycle: Move `connect_pipeline` from standalone function to `ResourcePool` method commit fc2714a Author: Jon Clucas <[email protected]> Date: Tue Jul 16 16:16:58 2024 -0400 :recycle: Clarify `ResourcePool.gather_pipes` commit 588df00 Author: Jon Clucas <[email protected]> Date: Tue Jul 16 15:36:24 2024 -0400 :recycle: Typehint `StratPool.append_name` commit 6207348 Author: Jon Clucas <[email protected]> Date: Mon Jul 15 17:41:51 2024 -0400 :zap: Replace some `deepcopy` calls commit abb4809 Author: Jon Clucas <[email protected]> Date: Mon Jul 15 16:13:39 2024 -0400 :art: Type `connect_pipeline` commit 5ea19ef Author: Jon Clucas <[email protected]> Date: Mon Jul 15 15:33:11 2024 -0400 :twisted_rightwards_arrow: Merge `develop` into `engine/(th)r(e)esources` Squashed commit of the following: commit a40c449 Merge: 6b6f7f4 9ba084e Author: Jon Clucas <[email protected]> Date: Mon Jul 15 13:16:14 2024 -0400 ⬆️ Bump zipp from 3.16.0 to 3.19.1 (#2132) commit 9ba084e Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue Jul 9 18:35:58 2024 +0000 :arrow_up: Bump zipp from 3.16.0 to 3.19.1 Bumps [zipp](https://github.com/jaraco/zipp) from 3.16.0 to 3.19.1. - [Release notes](https://github.com/jaraco/zipp/releases) - [Changelog](https://github.com/jaraco/zipp/blob/main/NEWS.rst) - [Commits](jaraco/zipp@v3.16.0...v3.19.1) --- updated-dependencies: - dependency-name: zipp dependency-type: direct:production ... Signed-off-by: dependabot[bot] <[email protected]> commit 6b6f7f4 Merge: 5ce5d37 979b0a9 Author: Jon Clucas <[email protected]> Date: Fri Jul 12 13:27:18 2024 -0400 🐛 Fix `get_scan_params` (#2129) commit 979b0a9 Author: Jon Clucas <[email protected]> Date: Tue Jul 9 09:39:03 2024 -0400 :pencil2: Fix f-string missing `f` commit af65a2e Author: Jon Clucas <[email protected]> Date: Mon Jul 8 17:01:06 2024 -0400 :bug: Fix circular import commit f194377 Author: Jon Clucas <[email protected]> Date: Mon Jul 8 16:07:15 2024 -0400 :recycle: Exclusively use custom `Function` Nodes + :rotating_light: Lint commit c7819d1 Author: Jon Clucas <[email protected]> Date: Mon Jul 8 14:16:53 2024 -0400 :art: Remove unnecessary initializations commit b013ccc Author: Jon Clucas <[email protected]> Date: Mon Jul 8 13:54:48 2024 -0400 :package: Init `Function` commit b19907a Author: Jon Clucas <[email protected]> Date: Mon Jul 8 13:40:19 2024 -0400 :bug: Use C-PAC Function node commit 7d6f0ee Author: Jon Clucas <[email protected]> Date: Mon Jul 8 12:59:12 2024 -0400 :pencil2: Fix TR capitalization commit 3ebb9f4 Author: Jon Clucas <[email protected]> Date: Mon Jul 8 12:54:48 2024 -0400 :white_check_mark: Add tests for `fetch` refactor [rebuild base-lite] [rebuild base-standard] [run reg-suite] commit 6a5b723 Author: Jon Clucas <[email protected]> Date: Mon Jul 8 12:01:02 2024 -0400 :recycle: DRY params, sub, scan commit c5c39b0 Author: Jon Clucas <[email protected]> Date: Mon Jul 8 10:49:11 2024 -0400 :bug: Tell Nipype to import typehint type [run reg-suite] commit 52aa366 Author: Jon Clucas <[email protected]> Date: Mon Jul 8 10:05:09 2024 -0400 :recycle: DRY `fetch_and_convert`|`fetch`|`check`|`check2` [run reg-suite] commit ddf2103 Author: Jon Clucas <[email protected]> Date: Sat Jul 6 00:27:06 2024 -0400 :construction: WIP :bug: Fix `get_scan_params` [run reg-suite] commit 17257e3 Author: Jon Clucas <[email protected]> Date: Fri Jul 5 14:26:27 2024 -0400 :recycle: Dedupe function node imports commit 74c0950 Author: Jon Clucas <[email protected]> Date: Fri Jul 5 09:56:33 2024 -0400 :bug: Import local variables in `get_scan_params` [run reg-suite] commit e64309c Author: Jon Clucas <[email protected]> Date: Wed Jul 3 18:15:43 2024 -0400 :bug: Fix import (probably merge error) [run reg-suite] commit faf8ab9 Author: Jon Clucas <[email protected]> Date: Mon Jul 15 14:44:58 2024 -0400 :white_check_mark: Check if `bids-examples` is empty before moving on commit e25479f Author: Jon Clucas <[email protected]> Date: Mon Jul 15 14:19:59 2024 -0400 :bug: Fix auto-`quick_single` commit 90d96b1 Author: Jon Clucas <[email protected]> Date: Mon Jul 15 14:10:55 2024 -0400 :white_check_mark: Update test for new `Resource` class commit c22db11 Author: Jon Clucas <[email protected]> Date: Mon Jul 15 14:02:58 2024 -0400 :bug: Remove extra curly braces commit 4280fc6 Author: Jon Clucas <[email protected]> Date: Mon Jul 15 13:47:15 2024 -0400 :recycle: Dedupe loop through `self.node_blocks.items()` commit aaa37a9 Author: Jon Clucas <[email protected]> Date: Mon Jul 15 13:14:09 2024 -0400 :recycle: Move `StratDict().rpool[json]` to `StratDict().json` commit 1058c94 Author: Jon Clucas <[email protected]> Date: Mon Jul 15 11:55:46 2024 -0400 :art: Define `_Pool.__contains__` and `Resource.__contains__` commit bb6cbae Author: Jon Clucas <[email protected]> Date: Mon Jul 15 10:01:42 2024 -0400 :bug: Fix circular import commit 1b11462 Author: Jon Clucas <[email protected]> Date: Fri Jul 12 22:14:15 2024 -0400 :technologist: Add `__repr__` method to `Resource` commit 02d1a14 Author: Jon Clucas <[email protected]> Date: Fri Jul 12 22:06:37 2024 -0400 :white_check_mark: Remove dir instead of file commit b1f16ed Author: Jon Clucas <[email protected]> Date: Fri Jul 12 22:02:47 2024 -0400 :art: Type `ResourcePool.get_strats` commit ec32f34 Author: Jon Clucas <[email protected]> Date: Fri Jul 12 20:52:33 2024 -0400 :pencil2: Pipe inside quotation marks commit 199bd60 Author: Jon Clucas <[email protected]> Date: Fri Jul 12 16:06:49 2024 -0400 :white_check_mark: `pytest.Cache.makedir` → `pytest.Cache.mkdir` commit 1cc3914 Author: Jon Clucas <[email protected]> Date: Fri Jul 12 15:37:58 2024 -0400 :art: :technologist: Clarify typing commit 4d9934d Author: Jon Clucas <[email protected]> Date: Thu Jul 11 17:37:13 2024 -0400 :construction: WIP :recycle: Fix `StratPool.__init__` commit 98c8bb8 Author: Jon Clucas <[email protected]> Date: Thu Jul 11 14:13:24 2024 -0400 :construction: :recycle: Continue updating calls to `ResourcePool` methods commit 3af1407 Author: Jon Clucas <[email protected]> Date: Thu Jul 11 10:48:11 2024 -0400 :white_check_mark: Use abspath for BIDS-examples commit 87fb1c0 Author: Jon Clucas <[email protected]> Date: Thu Jul 11 10:37:20 2024 -0400 :white_check_mark: Skip NHP configs if no torch installed commit 8a24441 Author: Jon Clucas <[email protected]> Date: Thu Jul 11 10:28:45 2024 -0400 :recycle: Pass `part_id` through to `DataPaths` commit 3c2220d Author: Jon Clucas <[email protected]> Date: Thu Jul 11 10:23:18 2024 -0400 :pencil2: Import Path for signature commit 515b791 Author: Jon Clucas <[email protected]> Date: Thu Jul 11 10:20:55 2024 -0400 :recycle: EmptyConfiguration → Preconfiguration('blank') commit f1f7705 Author: Jon Clucas <[email protected]> Date: Wed Jul 10 17:06:06 2024 -0400 :recycle: Finish moving `create_func_datasource` from func to method commit bbf3e97 Author: Jon Clucas <[email protected]> Date: Wed Jul 10 17:01:03 2024 -0400 :recycle: Fold `initiate_rpool` into `ResourcePool.__init__` commit 3334a2a Author: Jon Clucas <[email protected]> Date: Wed Jul 10 13:00:10 2024 -0400 :construction_worker: Replace non-`\w`-non-dot characters with hyphens commit 646f49d Author: Jon Clucas <[email protected]> Date: Wed Jul 10 12:52:05 2024 -0400 :white_check_mark: Restore engine unit tests commit 7a76603 Author: Jon Clucas <[email protected]> Date: Wed Jul 10 12:51:42 2024 -0400 :construction: WIP :recycle: Move `ResourcePool` init from functions to methods commit 06612ff Author: Jon Clucas <[email protected]> Date: Tue Jul 9 16:38:34 2024 -0400 :white_check_mark: Move BIDS examples to a fixture commit 821bcaa Author: Jon Clucas <[email protected]> Date: Mon Jul 8 17:20:09 2024 -0400 :construction: Split ResourcePool into three classes with docstrings commit b414d17 Author: Jon Clucas <[email protected]> Date: Mon Jul 8 17:15:27 2024 -0400 :truck: Move engine resources to own submodule
FCP-INDI · Aug 6, 2024 · ae29bb1 · ae29bb1
1 parent 384f1e7
commit ae29bb1
Show file tree

Hide file tree

Showing 47 changed files with 4,080 additions and 3,948 deletions.
diff --git a/.circleci/main.yml b/.circleci/main.yml
@@ -68,7 +68,9 @@ commands:
     steps:
       - run:
           name: Getting Sample BIDS Data
-          command: git clone https://github.com/bids-standard/bids-examples.git
+          command: |
+            mkdir -p /home/circleci/project/dev/circleci_data/.pytest_cache/d/bids-examples
+            git clone https://github.com/bids-standard/bids-examples.git /home/circleci/project/dev/circleci_data/.pytest_cache/d/bids-examples
   get-singularity:
     parameters:
       version:
@@ -156,7 +158,7 @@ commands:
             then
               TAG=nightly
             else
-              TAG="${CIRCLE_BRANCH//\//_}"
+              TAG=`echo ${CIRCLE_BRANCH} | sed 's/[^a-zA-Z0-9._]/-/g'`
             fi
             DOCKER_TAG="ghcr.io/${CIRCLE_PROJECT_USERNAME,,}/${CIRCLE_PROJECT_REPONAME,,}:${TAG,,}"
             if [[ -n "<< parameters.variant >>" ]]
@@ -172,7 +174,7 @@ commands:
           name: Testing Singularity installation
           command: |
             pip install -r dev/circleci_data/requirements.txt
-            coverage run -m pytest --junitxml=test-results/junit.xml --continue-on-collection-errors dev/circleci_data/test_install.py
+            coverage run -m pytest --capture=no --junitxml=test-results/junit.xml --continue-on-collection-errors dev/circleci_data/test_install.py
 
 jobs:
   combine-coverage:

diff --git a/.github/workflows/build_C-PAC.yml b/.github/workflows/build_C-PAC.yml
@@ -42,7 +42,7 @@ jobs:
           GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-)
           if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]]
           then
-            TAG=${GITHUB_BRANCH//\//_}
+            TAG=`echo ${GITHUB_BRANCH} | sed 's/[^a-zA-Z0-9._]/-/g'`
             DOCKERFILE=.github/Dockerfiles/C-PAC.develop$VARIANT-$OS.Dockerfile
           elif [[ $GITHUB_BRANCH == 'develop' ]]
           then

diff --git a/.github/workflows/regression_test_full.yml b/.github/workflows/regression_test_full.yml
@@ -13,7 +13,7 @@ jobs:
           GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-)
           if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]]
           then
-            TAG=${GITHUB_BRANCH//\//_}
+            TAG=`echo ${GITHUB_BRANCH} | sed 's/[^a-zA-Z0-9._]/-/g'`
           elif [[ $GITHUB_BRANCH == 'develop' ]]
           then
             TAG=nightly

diff --git a/.github/workflows/regression_test_lite.yml b/.github/workflows/regression_test_lite.yml
@@ -37,7 +37,7 @@ jobs:
         run: |
           if [[ ! $GITHUB_REF_NAME == 'main' ]] && [[ ! $GITHUB_REF_NAME == 'develop' ]]
           then
-            TAG=${GITHUB_REF_NAME//\//_}
+            TAG=`echo ${GITHUB_REF_NAME} | sed 's/[^a-zA-Z0-9._]/-/g'`
           elif [[ $GITHUB_REF_NAME == 'develop' ]]
           then
             TAG=nightly

diff --git a/.github/workflows/smoke_test_participant.yml b/.github/workflows/smoke_test_participant.yml
@@ -68,7 +68,7 @@ jobs:
           GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-)
           if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]]
           then
-            TAG=${GITHUB_BRANCH//\//_}
+            TAG=`echo ${GITHUB_BRANCH} | sed 's/[^a-zA-Z0-9._]/-/g'`
           elif [[ $GITHUB_BRANCH == 'develop' ]]
           then
             TAG=nightly
@@ -133,7 +133,7 @@ jobs:
           GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-)
           if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]]
           then
-            TAG=${GITHUB_BRANCH//\//_}
+            TAG=`echo ${GITHUB_BRANCH} | sed 's/[^a-zA-Z0-9._]/-/g'`
           elif [[ $GITHUB_BRANCH == 'develop' ]]
           then
             TAG=nightly
@@ -192,7 +192,7 @@ jobs:
           GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-)
           if [[ ! $GITHUB_BRANCH == 'main' ]] && [[ ! $GITHUB_BRANCH == 'develop' ]]
           then
-            TAG=${GITHUB_BRANCH//\//_}
+            TAG=`echo ${GITHUB_BRANCH} | sed 's/[^a-zA-Z0-9._]/-/g'`
           elif [[ $GITHUB_BRANCH == 'develop' ]]
           then
             TAG=nightly

diff --git a/.ruff.toml b/.ruff.toml
@@ -13,6 +13,7 @@ external = ["T20"]  # Don't autoremove 'noqa` comments for these rules
 "CPAC/utils/sklearn.py" = ["RUF003"]
 "CPAC/utils/tests/old_functions.py" = ["C", "D", "E", "EM", "PLW", "RET"]
 "CPAC/utils/utils.py" = ["T201"]  # until `repickle` is removed
+"dev/circleci_data/conftest.py" = ["F401"]
 "setup.py" = ["D1"]
 
 [lint.flake8-import-conventions.extend-aliases]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Moved `pygraphviz` from requirements to `graphviz` optional dependencies group.
+- Split `ResourcePool` into three classes: `Resource`, `ResourcePool`, and `StratPool`.
 
 ### Fixed
 

diff --git a/CPAC/alff/alff.py b/CPAC/alff/alff.py
@@ -22,7 +22,7 @@
 
 from CPAC.alff.utils import get_opt_string
 from CPAC.pipeline import nipype_pipeline_engine as pe
-from CPAC.pipeline.nodeblock import nodeblock
+from CPAC.pipeline.engine.nodeblock import nodeblock
 from CPAC.registration.registration import apply_transform
 from CPAC.utils.interfaces import Function
 from CPAC.utils.utils import check_prov_for_regtool

diff --git a/CPAC/anat_preproc/anat_preproc.py b/CPAC/anat_preproc/anat_preproc.py
@@ -34,7 +34,7 @@
     wb_command,
 )
 from CPAC.pipeline import nipype_pipeline_engine as pe
-from CPAC.pipeline.nodeblock import nodeblock
+from CPAC.pipeline.engine.nodeblock import nodeblock
 from CPAC.utils.interfaces import Function
 from CPAC.utils.interfaces.fsl import Merge as fslMerge
 

diff --git a/CPAC/conftest.py b/CPAC/conftest.py
@@ -0,0 +1,32 @@
+# Copyright (C) 2024  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+"""Global pytest configuration."""
+
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def bids_examples(cache: pytest.Cache) -> Path:
+    """Get cached example BIDS directories."""
+    bids_dir = cache.mkdir("bids-examples").absolute()
+    if not (bids_dir.exists() and list(bids_dir.iterdir())):
+        from git import Repo
+
+        Repo.clone_from("https://github.com/bids-standard/bids-examples.git", bids_dir)
+    return bids_dir
diff --git a/CPAC/distortion_correction/distortion_correction.py b/CPAC/distortion_correction/distortion_correction.py
@@ -32,7 +32,7 @@
     run_fsl_topup,
 )
 from CPAC.pipeline import nipype_pipeline_engine as pe
-from CPAC.pipeline.nodeblock import nodeblock
+from CPAC.pipeline.engine.nodeblock import nodeblock
 from CPAC.utils import function
 from CPAC.utils.datasource import match_epi_fmaps
 from CPAC.utils.interfaces.function import Function
@@ -438,11 +438,6 @@ def distcor_blip_afni_qwarp(wf, cfg, strat_pool, pipe_num, opt=None):
     node, out = strat_pool.get_data("pe-direction")
     wf.connect(node, out, match_epi_fmaps_node, "bold_pedir")
 
-    # interface = {'bold': (match_epi_fmaps_node, 'opposite_pe_epi'),
-    #             'desc-brain_bold': 'opposite_pe_epi_brain'}
-    # wf, strat_pool = wrap_block([bold_mask_afni, bold_masking],
-    #                            interface, wf, cfg, strat_pool, pipe_num, opt)
-
     func_get_brain_mask = pe.Node(
         interface=preprocess.Automask(), name=f"afni_mask_opposite_pe_{pipe_num}"
     )
@@ -530,10 +525,6 @@ def distcor_blip_afni_qwarp(wf, cfg, strat_pool, pipe_num, opt=None):
     wf.connect(node, out, undistort_func_mean, "reference_image")
     wf.connect(convert_afni_warp, "ants_warp", undistort_func_mean, "transforms")
 
-    # interface = {'desc-preproc_bold': (undistort_func_mean, 'output_image')}
-    # wf, strat_pool = wrap_block([bold_mask_afni],
-    #                            interface, wf, cfg, strat_pool, pipe_num, opt)
-
     remask = pe.Node(
         interface=preprocess.Automask(), name=f"afni_remask_boldmask_{pipe_num}"
     )
@@ -764,7 +755,7 @@ def distcor_blip_fsl_topup(wf, cfg, strat_pool, pipe_num, opt=None):
     wf.connect(run_topup, "out_jacs", vnum_base, "jac_matrix_list")
     wf.connect(run_topup, "out_warps", vnum_base, "warp_field_list")
 
-    mean_bold = strat_pool.node_data("sbref")
+    mean_bold = strat_pool.get_data("sbref")
 
     flirt = pe.Node(interface=fsl.FLIRT(), name="flirt")
     flirt.inputs.dof = 6

diff --git a/CPAC/func_preproc/func_ingress.py b/CPAC/func_preproc/func_ingress.py
@@ -14,12 +14,21 @@
 
 # You should have received a copy of the GNU Lesser General Public
 # License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
-from CPAC.utils.datasource import create_func_datasource, ingress_func_metadata
+"""Ingress functional data for preprocessing."""
+
+from CPAC.utils.strategy import Strategy
 
 
 def connect_func_ingress(
-    workflow, strat_list, c, sub_dict, subject_id, input_creds_path, unique_id=None
+    workflow,
+    strat_list: list[Strategy],
+    c,
+    sub_dict,
+    subject_id,
+    input_creds_path,
+    unique_id=None,
 ):
+    """Connect functional ingress workflow."""
     for num_strat, strat in enumerate(strat_list):
         if "func" in sub_dict:
             func_paths_dict = sub_dict["func"]
@@ -31,7 +40,9 @@ def connect_func_ingress(
         else:
             workflow_name = f"func_gather_{unique_id}_{num_strat}"
 
-        func_wf = create_func_datasource(func_paths_dict, workflow_name)
+        func_wf = strat._resource_pool.create_func_datasource(
+            func_paths_dict, workflow_name
+        )
 
         func_wf.inputs.inputnode.set(
             subject=subject_id,
@@ -47,8 +58,6 @@ def connect_func_ingress(
             }
         )
 
-        (workflow, strat.rpool, diff, blip, fmap_rp_list) = ingress_func_metadata(
-            workflow, c, strat.rpool, sub_dict, subject_id, input_creds_path, unique_id
-        )
+        diff, blip, fmap_rp_list = strat.rpool.ingress_func_metadata()
 
-    return (workflow, diff, blip, fmap_rp_list)
+    return strat.rpool.wf, diff, blip, fmap_rp_list
diff --git a/CPAC/func_preproc/func_motion.py b/CPAC/func_preproc/func_motion.py
@@ -31,7 +31,7 @@
     motion_power_statistics,
 )
 from CPAC.pipeline import nipype_pipeline_engine as pe
-from CPAC.pipeline.nodeblock import nodeblock
+from CPAC.pipeline.engine.nodeblock import nodeblock
 from CPAC.pipeline.schema import valid_options
 from CPAC.utils.interfaces.function import Function
 from CPAC.utils.utils import check_prov_for_motion_tool
@@ -830,7 +830,7 @@ def motion_estimate_filter(wf, cfg, strat_pool, pipe_num, opt=None):
     notch.inputs.lowpass_cutoff = opt.get("lowpass_cutoff")
     notch.inputs.filter_order = opt.get("filter_order")
 
-    movement_parameters = strat_pool.node_data("desc-movementParameters_motion")
+    movement_parameters = strat_pool.get_data("desc-movementParameters_motion")
     wf.connect(
         movement_parameters.node, movement_parameters.out, notch, "motion_params"
     )

diff --git a/CPAC/func_preproc/func_preproc.py b/CPAC/func_preproc/func_preproc.py
@@ -22,7 +22,7 @@
 
 from CPAC.func_preproc.utils import nullify
 from CPAC.pipeline import nipype_pipeline_engine as pe
-from CPAC.pipeline.nodeblock import nodeblock
+from CPAC.pipeline.engine.nodeblock import nodeblock
 from CPAC.utils.interfaces import Function
 from CPAC.utils.interfaces.ants import (
     AI,  # niworkflows
@@ -993,7 +993,7 @@ def bold_mask_fsl_afni(wf, cfg, strat_pool, pipe_num, opt=None):
     #     and this function has been changed.
 
     # CHANGES:
-    #     * Converted from a plain function to a CPAC.pipeline.nodeblock.NodeBlockFunction
+    #     * Converted from a plain function to a CPAC.pipeline.engine.nodeblock.NodeBlockFunction
     #     * Removed Registration version check
     #     * Hardcoded Registration parameters instead of loading epi_atlasbased_brainmask.json
     #     * Uses C-PAC's ``FSL-AFNI-brain-probseg`` template in place of ``templateflow.api.get("MNI152NLin2009cAsym", resolution=1, label="brain", suffix="probseg")``

diff --git a/CPAC/func_preproc/tests/test_preproc_connections.py b/CPAC/func_preproc/tests/test_preproc_connections.py
@@ -36,7 +36,6 @@
 )
 from CPAC.func_preproc.func_preproc import func_normalize
 from CPAC.nuisance.nuisance import choose_nuisance_blocks
-from CPAC.pipeline.cpac_pipeline import connect_pipeline
 from CPAC.pipeline.engine import ResourcePool
 from CPAC.pipeline.nipype_pipeline_engine import Workflow
 from CPAC.registration.registration import (
@@ -81,7 +80,7 @@
     "from-template_to-T1w_mode-image_desc-linear_xfm",
 ]
 
-NUM_TESTS = 48  # number of parameterizations to run for many-parameter tests
+NUM_TESTS = 8  # number of parameterizations to run for many-parameter tests
 
 
 def _filter_assertion_message(
@@ -268,7 +267,7 @@ def test_motion_filter_connections(
     if not rpool.check_rpool("desc-cleaned_bold"):
         pipeline_blocks += choose_nuisance_blocks(c, generate_only)
     wf = Workflow(re.sub(r"[\[\]\-\:\_ \'\",]", "", str(rpool)))
-    connect_pipeline(wf, c, rpool, pipeline_blocks)
+    rpool.connect_pipeline(wf, c, pipeline_blocks)
     # Check that filtering is happening as expected
     filter_switch_key = [
         "functional_preproc",