From fb76b52f77e50f2ce15237143e4e08851ebba402 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 16 Mar 2021 20:05:57 +0000 Subject: [PATCH 1/9] :memo: Update usage from helpstring --- README.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 09dc61a7..4834b225 100644 --- a/README.rst +++ b/README.rst @@ -29,8 +29,7 @@ Usage .. code-block:: shell cpac --help - usage: cpac [-h] [--version] [-o [OPT [OPT ...]]] - [-B [CUSTOM_BINDING [CUSTOM_BINDING ...]]] + usage: cpac [-h] [--version] [-o OPT] [-B CUSTOM_BINDING] [--platform {docker,singularity}] [--image IMAGE] [--tag TAG] [--working_dir PATH] [-v] [-vv] {run,group,utils,pull,upgrade,crash} ... @@ -64,13 +63,13 @@ Usage optional arguments: -h, --help show this help message and exit --version show program's version number and exit - -o [OPT [OPT ...]], --container_option [OPT [OPT ...]] + -o OPT, --container_option OPT parameters and flags to pass through to Docker or Singularity This flag can take multiple arguments so cannot be the final argument before the command argument (i.e., run or any other command that does not start with - or --) - -B [CUSTOM_BINDING [CUSTOM_BINDING ...]], --custom_binding [CUSTOM_BINDING [CUSTOM_BINDING ...]] + -B CUSTOM_BINDING, --custom_binding CUSTOM_BINDING directories to bind with a different path in the container than the real path of the directory. One or more pairs in the format: From cc3d73054ff7200fd4df70d18c5771368274aa30 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 9 Aug 2021 16:02:17 -0400 Subject: [PATCH 2/9] :bookmark: Bump version to 0.3.3 --- CHANGELOG.rst | 4 ++++ setup.cfg | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5dde25d6..de613aef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,10 @@ ========= Changelog ========= +`Version 0.3.3: Minimal configs `_ +======================================================================================== +* 🐛 Resolves issue where minimal configs would cause wrapper to crash + `Version 0.3.2: Pull / Upgrade `_ ======================================================================================== * ➖ Remove dependecy on Nipype diff --git a/setup.cfg b/setup.cfg index 7b7974ca..86b3f1ed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ classifiers = Operating System :: OS Independent Programming Language :: Python :: 3 Topic :: Scientific/Engineering :: Bio-Informatics -version = 0.3.2.post1 +version = 0.3.3 [options] zip_safe = False From dbedf4bde293429db4ec9f351d4b7acdb7ba0888 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 9 Aug 2021 16:12:21 -0400 Subject: [PATCH 3/9] :test_tube: Test loading minimal config --- tests/test_cpac_run.py | 12 ++++++++++-- tests/test_data/minimal.min.yml | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 tests/test_data/minimal.min.yml diff --git a/tests/test_cpac_run.py b/tests/test_cpac_run.py index 874c17b2..dedb62bc 100644 --- a/tests/test_cpac_run.py +++ b/tests/test_cpac_run.py @@ -8,6 +8,10 @@ from cpac.__main__ import run from CONSTANTS import args_before_after, set_commandline_args +MINIMAL_CONFIG = os.path.join( + os.path.dirname(__file__), 'test_data', 'minimal.min.yml' +) + @pytest.mark.parametrize('helpflag', ['--help', '-h']) @pytest.mark.parametrize('argsep', [' ', '=']) @@ -32,7 +36,10 @@ def run_test(argv): @pytest.mark.parametrize('argsep', [' ', '=']) -def test_run_test_config(argsep, tmp_path, platform=None, tag=None): +@pytest.mark.parametrize('pipeline_file', [None, MINIMAL_CONFIG]) +def test_run_test_config( + argsep, pipeline_file, tmp_path, platform=None, tag=None +): def run_test(argv): with mock.patch.object(sys, 'argv', argv): run() @@ -42,10 +49,11 @@ def run_test(argv): wd = tmp_path args = set_commandline_args(platform, tag, argsep) + pipeline = '' if pipeline_file is None else f' {pipeline_file}' argv = ( 'run ' f's3://fcp-indi/data/Projects/ABIDE/RawDataBIDS/NYU {wd} ' - 'test_config --participant_ndx=2' + f'test_config --participant_ndx=2{pipeline}' ) if len(args): before, after = args_before_after(argv, args) diff --git a/tests/test_data/minimal.min.yml b/tests/test_data/minimal.min.yml new file mode 100644 index 00000000..f5251a1a --- /dev/null +++ b/tests/test_data/minimal.min.yml @@ -0,0 +1,5 @@ +FROM: default + +pipeline_setup: + + pipeline_name: test-minimal-imports From 89ae91b39588c72c4d36cc865302528a4487c5f2 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 9 Aug 2021 17:22:46 -0400 Subject: [PATCH 4/9] :bug: Don't break on minimal pipeline config --- src/cpac/backends/platform.py | 18 +++++++++++++++++- tests/test_cpac_run.py | 5 ++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/cpac/backends/platform.py b/src/cpac/backends/platform.py index 849bcb3d..74932292 100644 --- a/src/cpac/backends/platform.py +++ b/src/cpac/backends/platform.py @@ -9,6 +9,7 @@ from contextlib import redirect_stderr from io import StringIO from tabulate import tabulate +from warnings import warn from cpac.helpers import cpac_read_crash, get_extra_arg_value from cpac.utils import Locals_to_bind, PermissionMode @@ -83,6 +84,7 @@ def _bind_volume(self, local, remote, mode): self.volumes[local] = [b] def _collect_config_binding(self, config, config_key): + config_binding = None if isinstance(config, str): if os.path.exists(config): path = os.path.dirname(config) @@ -93,7 +95,21 @@ def _collect_config_binding(self, config, config_key): f'yaml.dump(Configuration({config}).dict())"' ) config = yaml.safe_load(config) - return config.get('pipeline_setup', {}).get(config_key, {}).get('path') + pipeline_setup = config.get('pipeline_setup', {}) + minimal = pipeline_setup.get('FROM', False) + if isinstance(pipeline_setup, dict): + config_binding = pipeline_setup.get(config_key, {}).get('path') + else: + minimal = True + if minimal: + warn( + 'This run is using a minimal pipeline configuration. If this ' + 'configuration imports a configuration that requires paths to ' + 'be bound from your real environment to your container, you ' + 'need to bind those paths manually with the `-B` flag.', + UserWarning + ) + return config_binding def collect_config_bindings(self, config, **kwargs): kwargs['output_dir'] = kwargs.get( diff --git a/tests/test_cpac_run.py b/tests/test_cpac_run.py index dedb62bc..e6b4be4c 100644 --- a/tests/test_cpac_run.py +++ b/tests/test_cpac_run.py @@ -49,7 +49,10 @@ def run_test(argv): wd = tmp_path args = set_commandline_args(platform, tag, argsep) - pipeline = '' if pipeline_file is None else f' {pipeline_file}' + pipeline = '' if pipeline_file is None else ' '.join([ + ' --pipeline_file', + pipeline_file + ]) argv = ( 'run ' f's3://fcp-indi/data/Projects/ABIDE/RawDataBIDS/NYU {wd} ' From b45e5065b9a8f4f14301d229f19e822254212f1a Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Wed, 27 Oct 2021 11:43:48 -0400 Subject: [PATCH 5/9] :alien: Remove call to deprecated Singularity Hub --- CHANGELOG.rst | 4 ++++ src/cpac/backends/singularity.py | 11 ++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5dde25d6..05d87c3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,10 @@ ========= Changelog ========= +`Version 0.4.0: Goodbye Singularity Hub `_ +================================================================================================ +* 👽 Drop call to now-deprecated Singularity Hub + `Version 0.3.2: Pull / Upgrade `_ ======================================================================================== * ➖ Remove dependecy on Nipype diff --git a/src/cpac/backends/singularity.py b/src/cpac/backends/singularity.py index 2e2f0a55..71da6e29 100644 --- a/src/cpac/backends/singularity.py +++ b/src/cpac/backends/singularity.py @@ -68,19 +68,12 @@ def pull(self, force=False, **kwargs): else: # pragma: no cover try: self.image = Client.pull( - "shub://FCP-INDI/C-PAC", + "docker://fcpindi/c-pac:latest", force=force, pull_folder=pwd ) except Exception: - try: - self.image = Client.pull( - "docker://fcpindi/c-pac:latest", - force=force, - pull_folder=pwd - ) - except Exception: - raise OSError("Could not connect to Singularity") + raise OSError("Could not connect to Singularity") def _try_to_stream(self, args, stream_command='run', **kwargs): self._bindings_as_option() From 406e2428fb7c19d388c32dde5799beb8966af983 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Wed, 27 Oct 2021 11:46:55 -0400 Subject: [PATCH 6/9] :bookmark: Bump version to 0.4.0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7b7974ca..8aba9d0c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ classifiers = Operating System :: OS Independent Programming Language :: Python :: 3 Topic :: Scientific/Engineering :: Bio-Informatics -version = 0.3.2.post1 +version = 0.4.0 [options] zip_safe = False From ad23e635b27a2e37f1c862fbfd571b6bbec55881 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Wed, 27 Oct 2021 11:52:55 -0400 Subject: [PATCH 7/9] =?UTF-8?q?:construction=5Fworker:=20Upgrade=20actions?= =?UTF-8?q?/setup-python@v2=20=E2=86=90=20v1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test_cpac.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_cpac.yml b/.github/workflows/test_cpac.yml index 29735b56..0fb974d8 100644 --- a/.github/workflows/test_cpac.yml +++ b/.github/workflows/test_cpac.yml @@ -33,9 +33,11 @@ jobs: with: go-version: ${{ matrix.go }} - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} + - name: Install --upgrade pip setuptools wheel + run: python -m pip install --upgrade pip setuptools wheel - name: Set up Singularity if: ${{ matrix.platform == 'singularity' }} uses: eWaterCycle/setup-singularity@v5 From 1c7e355aac1666963e6dec11e96a39388b2e45ec Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Wed, 27 Oct 2021 13:18:25 -0400 Subject: [PATCH 8/9] :white_check_mark: Update installation test for better practices --- tests/test_cpac_installation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_cpac_installation.py b/tests/test_cpac_installation.py index e2e466c0..0fe39007 100644 --- a/tests/test_cpac_installation.py +++ b/tests/test_cpac_installation.py @@ -1,4 +1,5 @@ -from pip._internal.utils.misc import get_installed_distributions +"""Test if cpac is installed as expected""" +from pkg_resources import working_set from setuptools.config import read_configuration @@ -22,7 +23,7 @@ def test_requirements(): 'but not in setup.cfg' ) assert package_in_list( - req, requirements_list(get_installed_distributions()) + req, requirements_list(working_set) ), ( f'package {req} is in requirements.txt ' 'but not installed' From aa393c3b4f84486c48ae1ebbba5e7354d4c28ec8 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 1 Feb 2022 16:00:49 -0500 Subject: [PATCH 9/9] :construction_worker: Don't run tests on pull target --- .github/workflows/test_cpac.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_cpac.yml b/.github/workflows/test_cpac.yml index 0fb974d8..f82c94e1 100644 --- a/.github/workflows/test_cpac.yml +++ b/.github/workflows/test_cpac.yml @@ -8,7 +8,6 @@ on: paths-ignore: - README.rst pull_request: - pull_request_target: jobs: test_cpac: