From fd21adeaafb6875e4c95fd22b21dc207f6639a66 Mon Sep 17 00:00:00 2001 From: Camilo Velez Date: Tue, 9 Apr 2024 17:42:39 -0400 Subject: [PATCH] refactor: remove polus-plugins --- .gitignore | 5 - VERSION | 1 - noxfile.py | 26 - pyproject.toml | 63 -- src/polus/plugins/__init__.py | 65 -- src/polus/plugins/_plugins/VERSION | 1 - src/polus/plugins/_plugins/__init__.py | 0 src/polus/plugins/_plugins/_compat.py | 4 - .../plugins/_plugins/classes/__init__.py | 27 - .../plugins/_plugins/classes/plugin_base.py | 311 -------- .../_plugins/classes/plugin_classes.py | 472 ------------ src/polus/plugins/_plugins/cwl/__init__.py | 3 - src/polus/plugins/_plugins/cwl/base.cwl | 17 - src/polus/plugins/_plugins/cwl/cwl.py | 7 - src/polus/plugins/_plugins/gh.py | 65 -- src/polus/plugins/_plugins/io/__init__.py | 21 - src/polus/plugins/_plugins/io/_io.py | 596 -------------- .../plugins/_plugins/manifests/__init__.py | 15 - .../_plugins/manifests/manifest_utils.py | 210 ----- .../_plugins/models/PolusComputeSchema.json | 499 ------------ .../_plugins/models/PolusComputeSchema.ts | 102 --- src/polus/plugins/_plugins/models/__init__.py | 35 - .../models/pydanticv1/PolusComputeSchema.py | 137 ---- .../models/pydanticv1/WIPPPluginSchema.py | 233 ------ .../_plugins/models/pydanticv1/__init__.py | 0 .../_plugins/models/pydanticv1/compute.py | 28 - .../_plugins/models/pydanticv1/wipp.py | 79 -- .../models/pydanticv2/PolusComputeSchema.py | 136 ---- .../models/pydanticv2/WIPPPluginSchema.py | 241 ------ .../_plugins/models/pydanticv2/__init__.py | 0 .../_plugins/models/pydanticv2/compute.py | 28 - .../_plugins/models/pydanticv2/wipp.py | 79 -- .../models/wipp-plugin-manifest-schema.json | 726 ------------------ src/polus/plugins/_plugins/registry.py | 280 ------- src/polus/plugins/_plugins/registry_utils.py | 135 ---- src/polus/plugins/_plugins/update/__init__.py | 6 - src/polus/plugins/_plugins/update/_update.py | 116 --- src/polus/plugins/_plugins/utils.py | 17 - tests/__init__.py | 1 - tests/resources/b1.json | 77 -- tests/resources/b2.json | 76 -- tests/resources/b3.json | 76 -- tests/resources/g1.json | 78 -- tests/resources/g2.json | 77 -- tests/resources/g3.json | 77 -- tests/resources/omeconverter022.json | 45 -- tests/resources/omeconverter030.json | 70 -- tests/resources/target1.cwl | 32 - tests/test_cwl.py | 105 --- tests/test_io.py | 69 -- tests/test_manifests.py | 236 ------ tests/test_plugins.py | 198 ----- tests/test_version.py | 171 ----- to_clt.py | 108 --- to_ict.py | 99 --- 55 files changed, 6381 deletions(-) delete mode 100644 VERSION delete mode 100644 noxfile.py delete mode 100644 pyproject.toml delete mode 100644 src/polus/plugins/__init__.py delete mode 100644 src/polus/plugins/_plugins/VERSION delete mode 100644 src/polus/plugins/_plugins/__init__.py delete mode 100644 src/polus/plugins/_plugins/_compat.py delete mode 100644 src/polus/plugins/_plugins/classes/__init__.py delete mode 100644 src/polus/plugins/_plugins/classes/plugin_base.py delete mode 100644 src/polus/plugins/_plugins/classes/plugin_classes.py delete mode 100644 src/polus/plugins/_plugins/cwl/__init__.py delete mode 100644 src/polus/plugins/_plugins/cwl/base.cwl delete mode 100644 src/polus/plugins/_plugins/cwl/cwl.py delete mode 100644 src/polus/plugins/_plugins/gh.py delete mode 100644 src/polus/plugins/_plugins/io/__init__.py delete mode 100644 src/polus/plugins/_plugins/io/_io.py delete mode 100644 src/polus/plugins/_plugins/manifests/__init__.py delete mode 100644 src/polus/plugins/_plugins/manifests/manifest_utils.py delete mode 100644 src/polus/plugins/_plugins/models/PolusComputeSchema.json delete mode 100644 src/polus/plugins/_plugins/models/PolusComputeSchema.ts delete mode 100644 src/polus/plugins/_plugins/models/__init__.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv1/PolusComputeSchema.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv1/WIPPPluginSchema.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv1/__init__.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv1/compute.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv1/wipp.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv2/PolusComputeSchema.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv2/WIPPPluginSchema.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv2/__init__.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv2/compute.py delete mode 100644 src/polus/plugins/_plugins/models/pydanticv2/wipp.py delete mode 100644 src/polus/plugins/_plugins/models/wipp-plugin-manifest-schema.json delete mode 100644 src/polus/plugins/_plugins/registry.py delete mode 100644 src/polus/plugins/_plugins/registry_utils.py delete mode 100644 src/polus/plugins/_plugins/update/__init__.py delete mode 100644 src/polus/plugins/_plugins/update/_update.py delete mode 100644 src/polus/plugins/_plugins/utils.py delete mode 100644 tests/__init__.py delete mode 100644 tests/resources/b1.json delete mode 100644 tests/resources/b2.json delete mode 100644 tests/resources/b3.json delete mode 100644 tests/resources/g1.json delete mode 100644 tests/resources/g2.json delete mode 100644 tests/resources/g3.json delete mode 100644 tests/resources/omeconverter022.json delete mode 100644 tests/resources/omeconverter030.json delete mode 100644 tests/resources/target1.cwl delete mode 100644 tests/test_cwl.py delete mode 100644 tests/test_io.py delete mode 100644 tests/test_manifests.py delete mode 100644 tests/test_plugins.py delete mode 100644 tests/test_version.py delete mode 100644 to_clt.py delete mode 100644 to_ict.py diff --git a/.gitignore b/.gitignore index a07072cae..b48f2fdf4 100644 --- a/.gitignore +++ b/.gitignore @@ -165,11 +165,6 @@ cython_debug/ # test data directory data -# local manifests -src/polus/plugins/_plugins/manifests/* - -# allow python scripts inside manifests dir -!src/polus/plugins/_plugins/manifests/*.py #macOS *.DS_Store diff --git a/VERSION b/VERSION deleted file mode 100644 index 17e51c385..000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.1 diff --git a/noxfile.py b/noxfile.py deleted file mode 100644 index dc282ac89..000000000 --- a/noxfile.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Nox automation file.""" - -from nox import Session, session - -python_versions = ["3.9"] - - -@session(python=["3.9"]) -def export_ts(session: Session) -> None: - """Export Pydantic model as TypeScript object.""" - session.install("-r", "requirements-dev.txt") - - session.run( - "datamodel-codegen", - "--input", - "./polus/_plugins/models/PolusComputeSchema.json", - "--output", - "./polus/_plugins/models/PolusComputeSchema.py", - ) - session.run( - "pydantic2ts", - "--module", - "./polus/_plugins/models/PolusComputeSchema.py", - "--output", - "./polus/_plugins/models/PolusComputeSchema.ts", - ) diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index d7f6d7118..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,63 +0,0 @@ -[tool.poetry] -authors = ["Nicholas Schaub ", "Camilo Velez "] -description = "Python API to configure and run Polus Plugins." -license = "License :: OSI Approved :: MIT License" -maintainers = ["Camilo Velez "] -name = "polus-plugins" -packages = [{include = "polus", from = "src"}] -readme = "README.md" -repository = "https://github.com/PolusAI/polus-plugins" -version = "0.1.1" - -[tool.poetry.dependencies] -python = ">=3.9, <3.12" - -click = "^8.1.3" -cwltool = "^3.1.20230513155734" -fsspec = "^2023.6.0" -pydantic = ">=1.10.0" -pygithub = "^1.58.2" -python-on-whales = "^0.68.0" -pyyaml = "^6.0" -tqdm = "^4.65.0" -validators = "^0.22.0" -xmltodict = "^0.13.0" - -[tool.poetry.group.dev.dependencies] -python = ">=3.9, <3.12" - -black = "^23.3.0" -bump2version = "^1.0.1" -datamodel-code-generator = "^0.23.0" -flake8 = "^6.0.0" -fsspec = "^2023.1.0" -mypy = "^1.4.0" -nox = "^2022.11.21" -poetry = "^1.3.2" -pre-commit = "^3.3.3" -pydantic = ">=1.10" -pytest = "^7.3.2" -pytest-benchmark = "^4.0.0" -pytest-cov = "^4.1.0" -pytest-sugar = "^0.9.7" -pytest-xdist = "^3.3.1" -python-on-whales = "^0.68.0" -pyyaml = "^6.0" -ruff = "^0.0.274" -tqdm = "^4.64.1" -xmltodict = "^0.13.0" - -[build-system] -build-backend = "poetry.core.masonry.api" -requires = ["poetry-core"] - -[tool.isort] -profile = "black" - -[tool.pytest.ini_options] -addopts = [ - "--import-mode=importlib", -] -markers = [ - "repo: marks tests that validate plugin.json files in local repo", -] diff --git a/src/polus/plugins/__init__.py b/src/polus/plugins/__init__.py deleted file mode 100644 index ac6a7c00e..000000000 --- a/src/polus/plugins/__init__.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Initialize polus-plugins module.""" - -import logging -from pathlib import Path -from typing import Union - -from polus.plugins._plugins.classes import ( - ComputePlugin, # pylint: disable=unused-import -) -from polus.plugins._plugins.classes import Plugin # pylint: disable=unused-import -from polus.plugins._plugins.classes import get_plugin # pylint: disable=unused-import -from polus.plugins._plugins.classes import list_plugins # pylint: disable=unused-import -from polus.plugins._plugins.classes import load_config # pylint: disable=unused-import -from polus.plugins._plugins.classes import refresh # pylint: disable=unused-import -from polus.plugins._plugins.classes import remove_all # pylint: disable=unused-import -from polus.plugins._plugins.classes import ( # pylint: disable=unused-import - remove_plugin, -) -from polus.plugins._plugins.classes import ( # pylint: disable=unused-import - submit_plugin, -) -from polus.plugins._plugins.update import ( # pylint: disable=unused-import - update_nist_plugins, -) -from polus.plugins._plugins.update import ( # pylint: disable=unused-import - update_polus_plugins, -) - -""" -Set up logging for the module -""" -logger = logging.getLogger("polus.plugins") - -with Path(__file__).parent.joinpath("_plugins/VERSION").open( - "r", - encoding="utf-8", -) as version_file: - VERSION = version_file.read().strip() - - -refresh() # calls the refresh method when library is imported - - -def __getattr__(name: str) -> Union[Plugin, ComputePlugin, list]: - if name == "list": - return list_plugins() - if name in list_plugins(): - return get_plugin(name) - if name in ["__version__", "VERSION"]: - return VERSION - msg = f"module '{__name__}' has no attribute '{name}'" - raise AttributeError(msg) - - -__all__ = [ - "refresh", - "submit_plugin", - "get_plugin", - "load_config", - "list_plugins", - "update_polus_plugins", - "update_nist_plugins", - "remove_all", - "remove_plugin", -] diff --git a/src/polus/plugins/_plugins/VERSION b/src/polus/plugins/_plugins/VERSION deleted file mode 100644 index 17e51c385..000000000 --- a/src/polus/plugins/_plugins/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.1 diff --git a/src/polus/plugins/_plugins/__init__.py b/src/polus/plugins/_plugins/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/polus/plugins/_plugins/_compat.py b/src/polus/plugins/_plugins/_compat.py deleted file mode 100644 index 190aa0fd3..000000000 --- a/src/polus/plugins/_plugins/_compat.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Compat of Pydantic.""" -import pydantic - -PYDANTIC_V2 = pydantic.VERSION.startswith("2.") diff --git a/src/polus/plugins/_plugins/classes/__init__.py b/src/polus/plugins/_plugins/classes/__init__.py deleted file mode 100644 index 2d4460519..000000000 --- a/src/polus/plugins/_plugins/classes/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Plugin classes and functions.""" - -from polus.plugins._plugins.classes.plugin_classes import PLUGINS -from polus.plugins._plugins.classes.plugin_classes import ComputePlugin -from polus.plugins._plugins.classes.plugin_classes import Plugin -from polus.plugins._plugins.classes.plugin_classes import _load_plugin -from polus.plugins._plugins.classes.plugin_classes import get_plugin -from polus.plugins._plugins.classes.plugin_classes import list_plugins -from polus.plugins._plugins.classes.plugin_classes import load_config -from polus.plugins._plugins.classes.plugin_classes import refresh -from polus.plugins._plugins.classes.plugin_classes import remove_all -from polus.plugins._plugins.classes.plugin_classes import remove_plugin -from polus.plugins._plugins.classes.plugin_classes import submit_plugin - -__all__ = [ - "Plugin", - "ComputePlugin", - "submit_plugin", - "get_plugin", - "refresh", - "list_plugins", - "remove_plugin", - "remove_all", - "load_config", - "_load_plugin", - "PLUGINS", -] diff --git a/src/polus/plugins/_plugins/classes/plugin_base.py b/src/polus/plugins/_plugins/classes/plugin_base.py deleted file mode 100644 index 02d142b7f..000000000 --- a/src/polus/plugins/_plugins/classes/plugin_base.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Methods for all plugin objects.""" -# pylint: disable=W1203, W0212, enable=W1201 -import enum -import json -import logging -import random -import signal -from pathlib import Path -from typing import Any -from typing import Optional -from typing import TypeVar -from typing import Union - -import fsspec -import yaml # type: ignore -from cwltool.context import RuntimeContext -from cwltool.factory import Factory -from cwltool.utils import CWLObjectType -from polus.plugins._plugins.cwl import CWL_BASE_DICT -from polus.plugins._plugins.io import input_to_cwl -from polus.plugins._plugins.io import io_to_yml -from polus.plugins._plugins.io import output_to_cwl -from polus.plugins._plugins.io import outputs_cwl -from polus.plugins._plugins.utils import name_cleaner -from python_on_whales import docker - -logger = logging.getLogger("polus.plugins") - -StrPath = TypeVar("StrPath", str, Path) - - -class IOKeyError(Exception): - """Raised when trying to set invalid I/O parameter.""" - - -class MissingInputValuesError(Exception): - """Raised when there are required input values that have not been set.""" - - -class BasePlugin: - """Base Class for Plugins.""" - - def _check_inputs(self) -> None: - """Check if all required inputs have been set.""" - _in = [x for x in self.inputs if x.required and not x.value] # type: ignore - if len(_in) > 0: - msg = f"{[x.name for x in _in]} are required inputs but have not been set" - raise MissingInputValuesError( - msg, # type: ignore - ) - - @property - def organization(self) -> str: - """Plugin container's organization.""" - return self.containerId.split("/")[0] - - def load_config(self, path: StrPath) -> None: - """Load configured plugin from file.""" - with Path(path).open(encoding="utf=8") as fw: - config = json.load(fw) - inp = config["inputs"] - out = config["outputs"] - for k, v in inp.items(): - if k in self._io_keys: - setattr(self, k, v) - for k, v in out.items(): - if k in self._io_keys: - setattr(self, k, v) - logger.debug(f"Loaded config from {path}") - - def run( - self, - gpus: Union[None, str, int] = "all", - **kwargs: Union[None, str, int], - ) -> None: - """Run plugin in Docker container.""" - self._check_inputs() - inp_dirs = [x for x in self.inputs if isinstance(x.value, Path)] - out_dirs = [x for x in self.outputs if isinstance(x.value, Path)] - - inp_dirs_dict = {x: f"/data/inputs/input{n}" for (n, x) in enumerate(inp_dirs)} - out_dirs_dict = { - x: f"/data/outputs/output{n}" for (n, x) in enumerate(out_dirs) - } - - mnts_in = [ - [f"type=bind,source={k},target={v},readonly"] # must be a list of lists - for (k, v) in inp_dirs_dict.items() - ] - mnts_out = [ - [f"type=bind,source={k},target={v}"] # must be a list of lists - for (k, v) in out_dirs_dict.items() - ] - - mnts = mnts_in + mnts_out - args = [] - - for i in self.inputs: - if i.value is not None: # do not include those with value=None - i._validate() - args.append(f"--{i.name}") - - if isinstance(i.value, Path): - args.append(inp_dirs_dict[str(i.value)]) - - elif isinstance(i.value, enum.Enum): - args.append(str(i.value._name_)) - - else: - args.append(str(i.value)) - - for o in self.outputs: - if o.value is not None: # do not include those with value=None - o._validate() - args.append(f"--{o.name}") - - if isinstance(o.value, Path): - args.append(out_dirs_dict[str(o.value)]) - - elif isinstance(o.value, enum.Enum): - args.append(str(o.value._name_)) - - else: - args.append(str(o.value)) - - random_int = random.randint(10, 99) # noqa: S311 # only for naming - container_name = f"polus{random_int}" - - def sig( - signal, # noqa # pylint: disable=W0613, W0621 - frame, # noqa # pylint: disable=W0613, W0621 - ) -> None: # signal handler to kill container when KeyboardInterrupt - logger.info(f"Exiting container {container_name}") - docker.kill(container_name) - - signal.signal( - signal.SIGINT, - sig, - ) # make of sig the handler for KeyboardInterrupt - if gpus is None: - logger.info( - f"""Running container without GPU. {self.__class__.__name__} - version {self.version!s}""", - ) - docker_ = docker.run( - self.containerId, - args, - name=container_name, - remove=True, - mounts=mnts, - **kwargs, - ) - print(docker_) # noqa - else: - logger.info( - f"""Running container with GPU: --gpus {gpus}. - {self.__class__.__name__} version {self.version!s}""", - ) - docker_ = docker.run( - self.containerId, - args, - gpus=gpus, - name=container_name, - remove=True, - mounts=mnts, - **kwargs, - ) - print(docker_) # noqa - - @property - def manifest(self) -> dict: - """Plugin manifest.""" - manifest_ = json.loads(self.json(exclude={"_io_keys", "versions", "id"})) - manifest_["version"] = manifest_["version"]["version"] - return manifest_ - - def __getattribute__(self, name: str) -> Any: # noqa - if name == "__class__": # pydantic v2 change - return super().__getattribute__(name) - if name != "_io_keys" and hasattr(self, "_io_keys") and name in self._io_keys: - value = self._io_keys[name].value - if isinstance(value, enum.Enum): - value = value.name - return value - - return super().__getattribute__(name) - - def __setattr__(self, name: str, value: Any) -> None: # noqa - if name == "_fs": - if not issubclass(type(value), fsspec.spec.AbstractFileSystem): - msg = "_fs must be an fsspec FileSystem" - raise ValueError(msg) - for i in self.inputs: - i._fs = value - for o in self.outputs: - o._fs = value - return - - if name != "_io_keys" and hasattr(self, "_io_keys"): - if name in self._io_keys: - logger.debug( - f"Value of {name} in {self.__class__.__name__} set to {value}", - ) - self._io_keys[name].value = value - return - msg = ( - f"Attempting to set {name} in " - "{self.__class__.__name__} but " - "{{name}} is not a valid I/O parameter" - ) - raise IOKeyError( - msg, - ) - - super().__setattr__(name, value) - - def _to_cwl(self) -> dict: - """Return CWL yml as dict.""" - cwl_dict = CWL_BASE_DICT - cwl_dict["inputs"] = {} - cwl_dict["outputs"] = {} - inputs = [input_to_cwl(x) for x in self.inputs] - inputs = inputs + [output_to_cwl(x) for x in self.outputs] - for inp in inputs: - cwl_dict["inputs"].update(inp) - outputs = [outputs_cwl(x) for x in self.outputs] - for out in outputs: - cwl_dict["outputs"].update(out) - cwl_dict["requirements"]["DockerRequirement"]["dockerPull"] = self.containerId - return cwl_dict - - def save_cwl(self, path: StrPath) -> Path: - """Save plugin as CWL command line tool.""" - if str(path).rsplit(".", maxsplit=1)[-1] != "cwl": - msg = "path must end in .cwl" - raise ValueError(msg) - with Path(path).open("w", encoding="utf-8") as file: - yaml.dump(self._to_cwl(), file) - return Path(path) - - @property - def _cwl_io(self) -> dict: - """Dict of I/O for CWL.""" - return { - x.name: io_to_yml(x) for x in self._io_keys.values() if x.value is not None - } - - def save_cwl_io(self, path: StrPath) -> Path: - """Save plugin's I/O values to yml file. - - To be used with CWL Command Line Tool. - """ - self._check_inputs() - if str(path).rsplit(".", maxsplit=1)[-1] != "yml": - msg = "path must end in .yml" - raise ValueError(msg) - with Path(path).open("w", encoding="utf-8") as file: - yaml.dump(self._cwl_io, file) - return Path(path) - - def run_cwl( - self, - cwl_path: Optional[StrPath] = None, - io_path: Optional[StrPath] = None, - ) -> Union[CWLObjectType, str, None]: - """Run configured plugin in CWL. - - Run plugin as a CWL command line tool after setting I/O values. - Two files will be generated: a CWL (`.cwl`) command line tool - and an I/O file (`.yml`). They will be generated in - current working directory if no paths are specified. Optional paths - for these files can be specified with arguments `cwl_path`, - and `io_path` respectively. - - Args: - cwl_path: [Optional] target path for `.cwl` file - io_path: [Optional] target path for `.yml` file - - """ - if not self.outDir: - msg = "" - raise ValueError(msg) - - if not cwl_path: - _p = Path.cwd().joinpath(name_cleaner(self.name) + ".cwl") - _cwl = self.save_cwl(_p) - else: - _cwl = self.save_cwl(cwl_path) - - if not io_path: - _p = Path.cwd().joinpath(name_cleaner(self.name) + ".yml") - self.save_cwl_io(_p) # saves io to make it visible to user - else: - self.save_cwl_io(io_path) # saves io to make it visible to user - - outdir_path = self.outDir.parent.relative_to(Path.cwd()) - r_c = RuntimeContext({"outdir": str(outdir_path)}) - fac = Factory(runtime_context=r_c) - cwl = fac.make(str(_cwl)) - return cwl(**self._cwl_io) # object's io dict is used instead of .yml file - - def __lt__(self, other: "BasePlugin") -> bool: - return self.version < other.version - - def __gt__(self, other: "BasePlugin") -> bool: - return other.version < self.version - - def __repr__(self) -> str: - return ( - f"{self.__class__.__name__}(name='{self.name}', version={self.version!s})" - ) diff --git a/src/polus/plugins/_plugins/classes/plugin_classes.py b/src/polus/plugins/_plugins/classes/plugin_classes.py deleted file mode 100644 index ca453c8c9..000000000 --- a/src/polus/plugins/_plugins/classes/plugin_classes.py +++ /dev/null @@ -1,472 +0,0 @@ -"""Classes for Plugin objects containing methods to configure, run, and save.""" -# pylint: disable=W1203, W0212, enable=W1201 -import json -import logging -import shutil -import uuid -from copy import deepcopy -from pathlib import Path -from typing import Any -from typing import Optional -from typing import Union - -from polus.plugins._plugins._compat import PYDANTIC_V2 -from polus.plugins._plugins.classes.plugin_base import BasePlugin -from polus.plugins._plugins.io._io import DuplicateVersionFoundError -from polus.plugins._plugins.io._io import Version -from polus.plugins._plugins.io._io import _in_old_to_new -from polus.plugins._plugins.io._io import _ui_old_to_new -from polus.plugins._plugins.manifests import InvalidManifestError -from polus.plugins._plugins.manifests import _load_manifest -from polus.plugins._plugins.manifests import validate_manifest -from polus.plugins._plugins.models import ComputeSchema -from polus.plugins._plugins.models import PluginUIInput -from polus.plugins._plugins.models import PluginUIOutput -from polus.plugins._plugins.models import WIPPPluginManifest -from polus.plugins._plugins.utils import cast_version -from polus.plugins._plugins.utils import name_cleaner -from pydantic import ConfigDict - -logger = logging.getLogger("polus.plugins") -PLUGINS: dict[str, dict] = {} -# PLUGINS = {"BasicFlatfieldCorrectionPlugin": -# {Version('0.1.4'): Path(<...>), Version('0.1.5'): Path(<...>)}. -# "VectorToLabel": {Version(...)}} - -""" -Paths and Fields -""" -# Location to store any discovered plugin manifests -_PLUGIN_DIR = Path(__file__).parent.parent.joinpath("manifests") - - -def refresh() -> None: - """Refresh the plugin list.""" - organizations = [ - x for x in _PLUGIN_DIR.iterdir() if x.name != "__pycache__" and x.is_dir() - ] # ignore __pycache__ - - PLUGINS.clear() - - for org in organizations: - for file in org.iterdir(): - if file.suffix == ".py": - continue - - try: - plugin = validate_manifest(file) - except InvalidManifestError: - logger.warning(f"Validation error in {file!s}") - except BaseException as exc: # pylint: disable=W0718 - logger.warning(f"Unexpected error {exc} with {file!s}") - raise exc - - else: - key = name_cleaner(plugin.name) - # Add version and path to VERSIONS - if key not in PLUGINS: - PLUGINS[key] = {} - if ( - plugin.version in PLUGINS[key] - and file != PLUGINS[key][plugin.version] - ): - msg = ( - "Found duplicate version of plugin" - f"{plugin.name} in {_PLUGIN_DIR}" - ) - raise DuplicateVersionFoundError( - msg, - ) - PLUGINS[key][plugin.version] = file - - -def list_plugins() -> list: - """List all local plugins.""" - output = list(PLUGINS.keys()) - output.sort() - return output - - -def _get_config(plugin: Union["Plugin", "ComputePlugin"], class_: str) -> dict: - if PYDANTIC_V2: - model_ = json.loads(plugin.model_dump_json()) - model_["_io_keys"] = deepcopy(plugin._io_keys) # type: ignore - else: - # ignore mypy if pydantic < 2.0.0 - model_ = plugin.dict() # type: ignore - # iterate over I/O to convert to dict - for io_name, io in model_["_io_keys"].items(): - if PYDANTIC_V2: - model_["_io_keys"][io_name] = json.loads(io.model_dump_json()) - # overwrite val if enum - if io.type.value == "enum": - model_["_io_keys"][io_name]["value"] = io.value.name # str - elif io["type"] == "enum": # pydantic V1 - val_ = io["value"].name # mapDirectory.raw - model_["_io_keys"][io_name]["value"] = val_.split(".")[-1] # raw - for inp in model_["inputs"]: - inp["value"] = None - model_["class"] = class_ - return model_ - - -class Plugin(WIPPPluginManifest, BasePlugin): - """WIPP Plugin Class. - - Contains methods to configure, run, and save plugins. - - Attributes: - versions: A list of local available versions for this plugin. - - Methods: - save_manifest(path): save plugin manifest to specified path - """ - - id: uuid.UUID # noqa: A003 - if PYDANTIC_V2: - model_config = ConfigDict(extra="allow", frozen=True) - else: - - class Config: # pylint: disable=R0903 - """Config.""" - - extra = "allow" - allow_mutation = False - - def __init__(self, _uuid: bool = True, **data: dict) -> None: - """Init a plugin object from manifest.""" - if _uuid: - data["id"] = uuid.uuid4() # type: ignore - else: - data["id"] = uuid.UUID(str(data["id"])) # type: ignore - - if not PYDANTIC_V2: # pydantic V1 - data["version"] = cast_version(data["version"]) - - super().__init__(**data) - - if not PYDANTIC_V2: # pydantic V1 - self.Config.allow_mutation = True - - self._io_keys = {i.name: i for i in self.inputs} - self._io_keys.update({o.name: o for o in self.outputs}) - - if not self.author: - warn_msg = ( - f"The plugin ({self.name}) is missing the author field. " - "This field is not required but should be filled in." - ) - logger.warning(warn_msg) - - @property - def versions(self) -> list: # cannot be in PluginMethods because PLUGINS lives here - """Return list of local versions of a Plugin.""" - return list(PLUGINS[name_cleaner(self.name)]) - - def to_compute( - self, - hardware_requirements: Optional[dict] = None, - ) -> type[ComputeSchema]: - """Convert WIPP Plugin object to Compute Plugin object.""" - data = deepcopy(self.manifest) - return ComputePlugin( - hardware_requirements=hardware_requirements, - _from_old=True, - **data, - ) - - def save_manifest( - self, - path: Union[str, Path], - hardware_requirements: Optional[dict] = None, - compute: bool = False, - ) -> None: - """Save plugin manifest to specified path.""" - if compute: - with Path(path).open("w", encoding="utf-8") as file: - self.to_compute( - hardware_requirements=hardware_requirements, - ).save_manifest(path) - else: - with Path(path).open("w", encoding="utf-8") as file: - dict_ = self.manifest - json.dump( - dict_, - file, - indent=4, - ) - - logger.debug(f"Saved manifest to {path}") - - def __setattr__(self, name: str, value: Any) -> None: # noqa: ANN401 - """Set I/O parameters as attributes.""" - BasePlugin.__setattr__(self, name, value) - - def save_config(self, path: Union[str, Path]) -> None: - """Save manifest with configured I/O parameters to specified path.""" - with Path(path).open("w", encoding="utf-8") as file: - json.dump(_get_config(self, "WIPP"), file, indent=4, default=str) - logger.debug(f"Saved config to {path}") - - def __repr__(self) -> str: - """Print plugin name and version.""" - return BasePlugin.__repr__(self) - - -class ComputePlugin(ComputeSchema, BasePlugin): - """Compute Plugin Class. - - Contains methods to configure, run, and save plugins. - - Attributes: - versions: A list of local available versions for this plugin. - - Methods: - save_manifest(path): save plugin manifest to specified path - """ - - if PYDANTIC_V2: - model_config = ConfigDict(extra="allow", frozen=True) - else: # pydantic V1 - - class Config: # pylint: disable=R0903 - """Config.""" - - extra = "allow" - allow_mutation = False - - def __init__( - self, - hardware_requirements: Optional[dict] = None, - _from_old: bool = False, - _uuid: bool = True, - **data: dict, - ) -> None: - """Init a plugin object from manifest.""" - if _uuid: - data["id"] = uuid.uuid4() # type: ignore - else: - data["id"] = uuid.UUID(str(data["id"])) # type: ignore - - if _from_old: - - def _convert_input(dict_: dict) -> dict: - dict_["type"] = _in_old_to_new(dict_["type"]) - return dict_ - - def _convert_output(dict_: dict) -> dict: - dict_["type"] = "path" - return dict_ - - def _ui_in(dict_: dict) -> PluginUIInput: # assuming old all ui input - # assuming format inputs. ___ - inp = dict_["key"].split(".")[-1] # e.g inpDir - try: - type_ = [x["type"] for x in data["inputs"] if x["name"] == inp][ - 0 - ] # get type from i/o - except IndexError: - type_ = "string" # default to string - except BaseException as exc: - raise exc - - dict_["type"] = _ui_old_to_new(type_) - return PluginUIInput(**dict_) - - def _ui_out(dict_: dict) -> PluginUIOutput: - new_dict_ = deepcopy(dict_) - new_dict_["name"] = "outputs." + new_dict_["name"] - new_dict_["type"] = _ui_old_to_new(new_dict_["type"]) - return PluginUIOutput(**new_dict_) - - data["inputs"] = [_convert_input(x) for x in data["inputs"]] # type: ignore - data["outputs"] = [ - _convert_output(x) for x in data["outputs"] - ] # type: ignore - data["pluginHardwareRequirements"] = {} - data["ui"] = [_ui_in(x) for x in data["ui"]] # type: ignore - data["ui"].extend( # type: ignore[attr-defined] - [_ui_out(x) for x in data["outputs"]], - ) - - if hardware_requirements: - for k, v in hardware_requirements.items(): - data["pluginHardwareRequirements"][k] = v - - data["version"] = cast_version(data["version"]) - super().__init__(**data) - self.Config.allow_mutation = True - self._io_keys = {i.name: i for i in self.inputs} - self._io_keys.update({o.name: o for o in self.outputs}) # type: ignore - - if not self.author: - warn_msg = ( - f"The plugin ({self.name}) is missing the author field. " - "This field is not required but should be filled in." - ) - logger.warning(warn_msg) - - @property - def versions(self) -> list: # cannot be in PluginMethods because PLUGINS lives here - """Return list of local versions of a Plugin.""" - return list(PLUGINS[name_cleaner(self.name)]) - - def __setattr__(self, name: str, value: Any) -> None: # noqa: ANN401 - """Set I/O parameters as attributes.""" - BasePlugin.__setattr__(self, name, value) - - def save_config(self, path: Union[str, Path]) -> None: - """Save configured manifest with I/O parameters to specified path.""" - with Path(path).open("w", encoding="utf-8") as file: - json.dump(_get_config(self, "Compute"), file, indent=4, default=str) - logger.debug(f"Saved config to {path}") - - def save_manifest(self, path: Union[str, Path]) -> None: - """Save plugin manifest to specified path.""" - with Path(path).open("w", encoding="utf-8") as file: - json.dump(self.manifest, file, indent=4) - logger.debug(f"Saved manifest to {path}") - - def __repr__(self) -> str: - """Print plugin name and version.""" - return BasePlugin.__repr__(self) - - -def _load_plugin( - manifest: Union[str, dict, Path], -) -> Union[Plugin, ComputePlugin]: - """Parse a manifest and return one of Plugin or ComputePlugin.""" - manifest = _load_manifest(manifest) - if "pluginHardwareRequirements" in manifest: # type: ignore[operator] - # Parse the manifest - plugin = ComputePlugin(**manifest) # type: ignore[arg-type] - else: - # Parse the manifest - plugin = Plugin(**manifest) # type: ignore[arg-type] - return plugin - - -def submit_plugin( - manifest: Union[str, dict, Path], -) -> Union[Plugin, ComputePlugin]: - """Parse a plugin and create a local copy of it. - - This function accepts a plugin manifest as a string, a dictionary (parsed - json), or a pathlib.Path object pointed at a plugin manifest. - - Args: - manifest: - A plugin manifest. It can be a url, a dictionary, - a path to a JSON file or a string that can be parsed as a dictionary - - Returns: - A Plugin object populated with information from the plugin manifest. - """ - plugin = validate_manifest(manifest) - plugin_name = name_cleaner(plugin.name) - - # Get Major/Minor/Patch versions - out_name = ( - plugin_name - + f"_M{plugin.version.major}m{plugin.version.minor}p{plugin.version.patch}.json" - ) - - # Save the manifest if it doesn't already exist in the database - organization = plugin.containerId.split("/")[0] - org_path = _PLUGIN_DIR.joinpath(organization.lower()) - org_path.mkdir(exist_ok=True, parents=True) - if not org_path.joinpath(out_name).exists(): - with org_path.joinpath(out_name).open("w", encoding="utf-8") as file: - if not PYDANTIC_V2: # pydantic V1 - manifest_ = plugin.dict() # type: ignore - manifest_["version"] = manifest_["version"]["version"] - else: # PYDANTIC V2 - manifest_ = json.loads(plugin.model_dump_json()) - json.dump(manifest_, file, indent=4) - - # Refresh plugins list - refresh() - return plugin - - -def get_plugin( - name: str, - version: Optional[str] = None, -) -> Union[Plugin, ComputePlugin]: - """Get a plugin with option to specify version. - - Return a plugin object with the option to specify a version. - The specified version's manifest must exist in manifests folder. - - Args: - name: Name of the plugin. - version: Optional version of the plugin, must follow semver. - - Returns: - Plugin object - """ - if version is None: - return _load_plugin(PLUGINS[name][max(PLUGINS[name])]) - if PYDANTIC_V2: - return _load_plugin(PLUGINS[name][Version(version)]) - return _load_plugin(PLUGINS[name][Version(**{"version": version})]) # Pydantic V1 - - -def load_config(config: Union[dict, Path, str]) -> Union[Plugin, ComputePlugin]: - """Load configured plugin from config file/dict.""" - if isinstance(config, (Path, str)): - with Path(config).open("r", encoding="utf-8") as file: - manifest_ = json.load(file) - elif isinstance(config, dict): - manifest_ = config - else: - msg = "config must be a dict, str, or a path" - raise TypeError(msg) - io_keys_ = manifest_["_io_keys"] - class_ = manifest_["class"] - manifest_.pop("class", None) - if class_ == "Compute": - plugin_ = ComputePlugin(_uuid=False, **manifest_) - elif class_ == "WIPP": - plugin_ = Plugin(_uuid=False, **manifest_) - else: - msg = "Invalid value of class" - raise ValueError(msg) - for key, value_ in io_keys_.items(): - val = value_["value"] - if val is not None: # exclude those values not set - setattr(plugin_, key, val) - return plugin_ - - -def remove_plugin(plugin: str, version: Optional[Union[str, list[str]]] = None) -> None: - """Remove plugin from the local database.""" - if version is None: - for plugin_version in PLUGINS[plugin]: - remove_plugin(plugin, plugin_version) - else: - if isinstance(version, list): - for version_ in version: - remove_plugin(plugin, version_) - return - if not PYDANTIC_V2: # pydantic V1 - if not isinstance(version, Version): - version_ = cast_version(version) - else: - version_ = version - else: # pydanitc V2 - version_ = Version(version) if not isinstance(version, Version) else version - path = PLUGINS[plugin][version_] - path.unlink() - refresh() - - -def remove_all() -> None: - """Remove all plugins from the local database.""" - organizations = [ - x for x in _PLUGIN_DIR.iterdir() if x.name != "__pycache__" and x.is_dir() - ] # ignore __pycache__ - logger.warning("Removing all plugins from local database") - for org in organizations: - shutil.rmtree(org) - refresh() diff --git a/src/polus/plugins/_plugins/cwl/__init__.py b/src/polus/plugins/_plugins/cwl/__init__.py deleted file mode 100644 index 966ef2d46..000000000 --- a/src/polus/plugins/_plugins/cwl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .cwl import CWL_BASE_DICT - -__all__ = ["CWL_BASE_DICT"] diff --git a/src/polus/plugins/_plugins/cwl/base.cwl b/src/polus/plugins/_plugins/cwl/base.cwl deleted file mode 100644 index 7a869228e..000000000 --- a/src/polus/plugins/_plugins/cwl/base.cwl +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.2 -class: CommandLineTool - -requirements: - DockerRequirement: - dockerPull: - InitialWorkDirRequirement: - listing: - - writable: true - entry: $(inputs.outDir) - InlineJavascriptRequirement: {} - -inputs: - -outputs: diff --git a/src/polus/plugins/_plugins/cwl/cwl.py b/src/polus/plugins/_plugins/cwl/cwl.py deleted file mode 100644 index 59a1163d9..000000000 --- a/src/polus/plugins/_plugins/cwl/cwl.py +++ /dev/null @@ -1,7 +0,0 @@ -from pathlib import Path - -import yaml # type: ignore - -PATH = Path(__file__) -with open(PATH.with_name("base.cwl"), "rb") as cwl_file: - CWL_BASE_DICT = yaml.full_load(cwl_file) diff --git a/src/polus/plugins/_plugins/gh.py b/src/polus/plugins/_plugins/gh.py deleted file mode 100644 index 80290b392..000000000 --- a/src/polus/plugins/_plugins/gh.py +++ /dev/null @@ -1,65 +0,0 @@ -"""GitHub utilties.""" -import logging -import os -from urllib.parse import urljoin - -import github - -from polus.plugins._plugins.classes import submit_plugin - -logger = logging.getLogger("polus.plugins") - -""" -Initialize the Github interface -""" - - -def _init_github(auth=None): - if auth is None: - # Try to get an auth key from an environment variable - auth = os.environ.get("GITHUB_AUTH", None) - - if auth is None: - gh = github.Github() - logger.warning("Initialized Github connection with no user token.") - return gh - else: - logger.debug("Found auth token in GITHUB_AUTH environment variable.") - - else: - logger.debug("Github auth token supplied as input.") - - gh = github.Github(login_or_token=auth) - logger.debug( - f"Initialized Github connection with token for user: {gh.get_user().login}" - ) - - return gh - - -def add_plugin_from_gh( - user: str, - branch: str, - plugin: str, - repo: str = "polus-plugins", - manifest_name: str = "plugin.json", -): - """Add plugin from GitHub. - - This function adds a plugin hosted on GitHub and returns a Plugin object. - - Args: - user: GitHub username - branch: GitHub branch - plugin: Plugin's name - repo: Name of GitHub repository, default is `polus-plugins` - manifest_name: Name of manifest file, default is `plugin.json` - - Returns: - A Plugin object populated with information from the plugin manifest. - """ - l1 = [user, repo, branch, plugin, manifest_name] - u = "/".join(l1) - url = urljoin("https://raw.githubusercontent.com", u) - logger.info("Adding %s" % url) - return submit_plugin(url, refresh=True) diff --git a/src/polus/plugins/_plugins/io/__init__.py b/src/polus/plugins/_plugins/io/__init__.py deleted file mode 100644 index 4687f4624..000000000 --- a/src/polus/plugins/_plugins/io/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Init IO module.""" - -from polus.plugins._plugins.io._io import Input -from polus.plugins._plugins.io._io import IOBase -from polus.plugins._plugins.io._io import Output -from polus.plugins._plugins.io._io import Version -from polus.plugins._plugins.io._io import input_to_cwl -from polus.plugins._plugins.io._io import io_to_yml -from polus.plugins._plugins.io._io import output_to_cwl -from polus.plugins._plugins.io._io import outputs_cwl - -__all__ = [ - "Input", - "Output", - "IOBase", - "Version", - "io_to_yml", - "outputs_cwl", - "input_to_cwl", - "output_to_cwl", -] diff --git a/src/polus/plugins/_plugins/io/_io.py b/src/polus/plugins/_plugins/io/_io.py deleted file mode 100644 index 6aff15a5f..000000000 --- a/src/polus/plugins/_plugins/io/_io.py +++ /dev/null @@ -1,596 +0,0 @@ -# type: ignore -# ruff: noqa: S101, A003 -# pylint: disable=no-self-argument, C0412 -"""Plugins I/O utilities.""" -import enum -import logging -import pathlib -import re -from functools import singledispatch -from functools import singledispatchmethod -from typing import Any -from typing import Optional -from typing import TypeVar -from typing import Union - -import fsspec -from polus.plugins._plugins._compat import PYDANTIC_V2 -from pydantic import BaseModel -from pydantic import Field -from pydantic import PrivateAttr - -if PYDANTIC_V2: - from typing import Annotated - - from pydantic import RootModel - from pydantic import StringConstraints - from pydantic import field_validator -else: - from pydantic import constr - from pydantic import validator - -logger = logging.getLogger("polus.plugins") - -""" -Enums for validating plugin input, output, and ui components. -""" -WIPP_TYPES = { - "collection": pathlib.Path, - "pyramid": pathlib.Path, - "csvCollection": pathlib.Path, - "genericData": pathlib.Path, - "stitchingVector": pathlib.Path, - "notebook": pathlib.Path, - "tensorflowModel": pathlib.Path, - "tensorboardLogs": pathlib.Path, - "pyramidAnnotation": pathlib.Path, - "integer": int, - "number": float, - "string": str, - "boolean": bool, - "array": str, - "enum": enum.Enum, - "path": pathlib.Path, -} - - -class InputTypes(str, enum.Enum): # wipp schema - """Enum of Input Types for WIPP schema.""" - - COLLECTION = "collection" - PYRAMID = "pyramid" - CSVCOLLECTION = "csvCollection" - GENERICDATA = "genericData" - STITCHINGVECTOR = "stitchingVector" - NOTEBOOK = "notebook" - TENSORFLOWMODEL = "tensorflowModel" - TENSORBOARDLOGS = "tensorboardLogs" - PYRAMIDANNOTATION = "pyramidAnnotation" - INTEGER = "integer" - NUMBER = "number" - STRING = "string" - BOOLEAN = "boolean" - ARRAY = "array" - ENUM = "enum" - - -class OutputTypes(str, enum.Enum): # wipp schema - """Enum for Output Types for WIPP schema.""" - - COLLECTION = "collection" - PYRAMID = "pyramid" - CSVCOLLECTION = "csvCollection" - GENERICDATA = "genericData" - STITCHINGVECTOR = "stitchingVector" - NOTEBOOK = "notebook" - TENSORFLOWMODEL = "tensorflowModel" - TENSORBOARDLOGS = "tensorboardLogs" - PYRAMIDANNOTATION = "pyramidAnnotation" - - -def _in_old_to_new(old: str) -> str: # map wipp InputType to compute schema's InputType - """Map an InputType from wipp schema to one of compute schema.""" - d = {"integer": "number", "enum": "string"} - if old in ["string", "array", "number", "boolean"]: - return old - if old in d: - return d[old] # integer or enum - return "path" # everything else - - -def _ui_old_to_new(old: str) -> str: # map wipp InputType to compute schema's UIType - """Map an InputType from wipp schema to a UIType of compute schema.""" - type_dict = { - "string": "text", - "boolean": "checkbox", - "number": "number", - "array": "text", - "integer": "number", - } - if old in type_dict: - return type_dict[old] - return "text" - - -FileSystem = TypeVar("FileSystem", bound=fsspec.spec.AbstractFileSystem) - - -class IOBase(BaseModel): # pylint: disable=R0903 - """Base Class for I/O arguments.""" - - type: Any = None - options: Optional[dict] = None - value: Optional[Any] = None - id_: Optional[Any] = None - _fs: Optional[FileSystem] = PrivateAttr( - default=None, - ) # type checking is done at plugin level - - def _validate(self) -> None: # pylint: disable=R0912 - value = self.value - - if value is None: - if self.required: - msg = f""" - The input value ({self.name}) is required, - but the value was not set.""" - raise TypeError( - msg, - ) - - return - - if self.type == InputTypes.ENUM: - try: - if isinstance(value, str): - value = enum.Enum(self.name, self.options["values"])[value] - elif not isinstance(value, enum.Enum): - raise ValueError - - except KeyError: - logging.error( - f""" - Value ({value}) is not a valid value - for the enum input ({self.name}). - Must be one of {self.options['values']}. - """, - ) - raise - else: - if isinstance(self.type, (InputTypes, OutputTypes)): # wipp - value = WIPP_TYPES[self.type](value) - else: - value = WIPP_TYPES[self.type.value]( - value, - ) # compute, type does not inherit from str - - if isinstance(value, pathlib.Path): - value = value.absolute() - if self._fs: - assert self._fs.exists( - str(value), - ), f"{value} is invalid or does not exist" - assert self._fs.isdir( - str(value), - ), f"{value} is not a valid directory" - else: - assert value.exists(), f"{value} is invalid or does not exist" - assert value.is_dir(), f"{value} is not a valid directory" - - super().__setattr__("value", value) - - def __setattr__(self, name: str, value: Any) -> None: # ruff: noqa: ANN401 - """Set I/O attributes.""" - if name not in ["value", "id", "_fs"]: - # Don't permit any other values to be changed - msg = f"Cannot set property: {name}" - raise TypeError(msg) - - super().__setattr__(name, value) - - if name == "value": - self._validate() - - -class Output(IOBase): # pylint: disable=R0903 - """Required until JSON schema is fixed.""" - - if PYDANTIC_V2: - name: Annotated[ - str, - StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), - ] = Field( - ..., - examples=["outputCollection"], - title="Output name", - ) - description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( - ..., - examples=["Output collection"], - title="Output description", - ) - else: - name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( - ..., - examples=["outputCollection"], - title="Output name", - ) - description: constr(regex=r"^(.*)$") = Field( - ..., - examples=["Output collection"], - title="Output description", - ) - type: OutputTypes = Field( - ..., - examples=["stitchingVector", "collection"], - title="Output type", - ) - - -class Input(IOBase): # pylint: disable=R0903 - """Required until JSON schema is fixed.""" - - if PYDANTIC_V2: - name: Annotated[ - str, - StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), - ] = Field( - ..., - description="Input name as expected by the plugin CLI", - examples=["inputImages", "fileNamePattern", "thresholdValue"], - title="Input name", - ) - description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( - ..., - examples=["Input Images"], - title="Input description", - ) - else: - name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( - ..., - description="Input name as expected by the plugin CLI", - examples=["inputImages", "fileNamePattern", "thresholdValue"], - title="Input name", - ) - description: constr(regex=r"^(.*)$") = Field( - ..., - examples=["Input Images"], - title="Input description", - ) - type: InputTypes - required: Optional[bool] = Field( - True, - description="Whether an input is required or not", - examples=[True], - title="Required input", - ) - - def __init__(self, **data) -> None: # ruff: noqa: ANN003 - """Initialize input.""" - super().__init__(**data) - - if self.description is None: - logger.warning( - f""" - The input ({self.name}) is missing the description field. - This field is not required but should be filled in. - """, - ) - - -def _check_version_number(value: Union[str, int]) -> bool: - if isinstance(value, int): - value = str(value) - if "-" in value: - value = value.split("-")[0] - if len(value) > 1 and value[0] == "0": - return False - return bool(re.match(r"^\d+$", value)) - - -if PYDANTIC_V2: - - class Version(RootModel): - """SemVer object.""" - - root: str - - @field_validator("root") - @classmethod - def semantic_version( - cls, - value, - ) -> Any: # ruff: noqa: ANN202, N805, ANN001 - """Pydantic Validator to check semver.""" - version = value.split(".") - - assert ( - len(version) == 3 # ruff: noqa: PLR2004 - ), f""" - Invalid version ({value}). Version must follow - semantic versioning (see semver.org)""" - if "-" in version[-1]: # with hyphen - idn = version[-1].split("-")[-1] - id_reg = re.compile("[0-9A-Za-z-]+") - assert bool( - id_reg.match(idn), - ), f"""Invalid version ({value}). - Version must follow semantic versioning (see semver.org)""" - - assert all( - map(_check_version_number, version), - ), f"""Invalid version ({value}). - Version must follow semantic versioning (see semver.org)""" - return value - - @property - def major(self): - """Return x from x.y.z .""" - return int(self.root.split(".")[0]) - - @property - def minor(self): - """Return y from x.y.z .""" - return int(self.root.split(".")[1]) - - @property - def patch(self): - """Return z from x.y.z .""" - if not self.root.split(".")[2].isdigit(): - msg = "Patch version is not a digit, comparison may not be accurate." - logger.warning(msg) - return self.root.split(".")[2] - return int(self.root.split(".")[2]) - - def __str__(self) -> str: - """Return string representation of Version object.""" - return self.root - - @singledispatchmethod - def __lt__(self, other: Any) -> bool: - """Compare if Version is less than other object.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - @singledispatchmethod - def __gt__(self, other: Any) -> bool: - """Compare if Version is less than other object.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - @singledispatchmethod - def __eq__(self, other: Any) -> bool: - """Compare if two Version objects are equal.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - def __hash__(self) -> int: - """Needed to use Version objects as dict keys.""" - return hash(self.root) - - def __repr__(self) -> str: - """Return string representation of Version object.""" - return self.root - - @Version.__eq__.register(str) # pylint: disable=no-member - def _(self, other): - return self == Version(other) - - @Version.__lt__.register(str) # pylint: disable=no-member - def _(self, other): - v = Version(other) - return self < v - - @Version.__gt__.register(str) # pylint: disable=no-member - def _(self, other): - v = Version(other) - return self > v - -else: # PYDANTIC_V1 - - class Version(BaseModel): - """SemVer object.""" - - version: str - - def __init__(self, version: str) -> None: - """Initialize Version object.""" - super().__init__(version=version) - - @validator("version") - def semantic_version( - cls, - value, - ): # ruff: noqa: ANN202, N805, ANN001 - """Pydantic Validator to check semver.""" - version = value.split(".") - - assert ( - len(version) == 3 # ruff: noqa: PLR2004 - ), f""" - Invalid version ({value}). Version must follow - semantic versioning (see semver.org)""" - if "-" in version[-1]: # with hyphen - idn = version[-1].split("-")[-1] - id_reg = re.compile("[0-9A-Za-z-]+") - assert bool( - id_reg.match(idn), - ), f"""Invalid version ({value}). - Version must follow semantic versioning (see semver.org)""" - - assert all( - map(_check_version_number, version), - ), f"""Invalid version ({value}). - Version must follow semantic versioning (see semver.org)""" - return value - - @property - def major(self): - """Return x from x.y.z .""" - return int(self.version.split(".")[0]) - - @property - def minor(self): - """Return y from x.y.z .""" - return int(self.version.split(".")[1]) - - @property - def patch(self): - """Return z from x.y.z .""" - if not self.version.split(".")[2].isdigit(): - msg = "Patch version is not a digit, comparison may not be accurate." - logger.warning(msg) - return self.version.split(".")[2] - return int(self.version.split(".")[2]) - - def __str__(self) -> str: - """Return string representation of Version object.""" - return self.version - - @singledispatchmethod - def __lt__(self, other: Any) -> bool: - """Compare if Version is less than other object.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - @singledispatchmethod - def __gt__(self, other: Any) -> bool: - """Compare if Version is less than other object.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - @singledispatchmethod - def __eq__(self, other: Any) -> bool: - """Compare if two Version objects are equal.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - def __hash__(self) -> int: - """Needed to use Version objects as dict keys.""" - return hash(self.version) - - @Version.__eq__.register(str) # pylint: disable=no-member - def _(self, other): - return self == Version(**{"version": other}) - - @Version.__lt__.register(str) # pylint: disable=no-member - def _(self, other): - v = Version(**{"version": other}) - return self < v - - @Version.__gt__.register(str) # pylint: disable=no-member - def _(self, other): - v = Version(**{"version": other}) - return self > v - - -@Version.__eq__.register(Version) # pylint: disable=no-member -def _(self, other): - return ( - other.major == self.major - and other.minor == self.minor - and other.patch == self.patch - ) - - -@Version.__lt__.register(Version) # pylint: disable=no-member -def _(self, other): - if other.major > self.major: - return True - if other.major == self.major: - if other.minor > self.minor: - return True - if other.minor == self.minor: - if other.patch > self.patch: - return True - return False - return False - return False - - -@Version.__gt__.register(Version) # pylint: disable=no-member -def _(self, other): - return other < self - - -class DuplicateVersionFoundError(Exception): - """Raise when two equal versions found.""" - - -CWL_INPUT_TYPES = { - "path": "Directory", # always Dir? Yes - "string": "string", - "number": "double", - "boolean": "boolean", - "genericData": "Directory", - "collection": "Directory", - "enum": "string", # for compatibility with workflows - "stitchingVector": "Directory", - # not yet implemented: array -} - - -def _type_in(inp: Input): - """Return appropriate value for `type` based on input type.""" - val = inp.type.value - req = "" if inp.required else "?" - - # NOT compatible with CWL workflows, ok in CLT - # if val == "enum": - # if input.required: - - # if val in CWL_INPUT_TYPES: - return CWL_INPUT_TYPES[val] + req if val in CWL_INPUT_TYPES else "string" + req - - -def input_to_cwl(inp: Input): - """Return dict of inputs for cwl.""" - return { - f"{inp.name}": { - "type": _type_in(inp), - "inputBinding": {"prefix": f"--{inp.name}"}, - }, - } - - -def output_to_cwl(out: Output): - """Return dict of output args for cwl for input section.""" - return { - f"{out.name}": { - "type": "Directory", - "inputBinding": {"prefix": f"--{out.name}"}, - }, - } - - -def outputs_cwl(out: Output): - """Return dict of output for `outputs` in cwl.""" - return { - f"{out.name}": { - "type": "Directory", - "outputBinding": {"glob": f"$(inputs.{out.name}.basename)"}, - }, - } - - -# -- I/O as arguments in .yml - - -@singledispatch -def _io_value_to_yml(io) -> Union[str, dict]: - return str(io) - - -@_io_value_to_yml.register -def _(io: pathlib.Path): - return {"class": "Directory", "location": str(io)} - - -@_io_value_to_yml.register -def _(io: enum.Enum): - return io.name - - -def io_to_yml(io): - """Return IO entry for yml file.""" - return _io_value_to_yml(io.value) diff --git a/src/polus/plugins/_plugins/manifests/__init__.py b/src/polus/plugins/_plugins/manifests/__init__.py deleted file mode 100644 index 9f1456e8a..000000000 --- a/src/polus/plugins/_plugins/manifests/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Initialize manifests module.""" - -from polus.plugins._plugins.manifests.manifest_utils import InvalidManifestError -from polus.plugins._plugins.manifests.manifest_utils import _error_log -from polus.plugins._plugins.manifests.manifest_utils import _load_manifest -from polus.plugins._plugins.manifests.manifest_utils import _scrape_manifests -from polus.plugins._plugins.manifests.manifest_utils import validate_manifest - -__all__ = [ - "InvalidManifestError", - "_load_manifest", - "validate_manifest", - "_error_log", - "_scrape_manifests", -] diff --git a/src/polus/plugins/_plugins/manifests/manifest_utils.py b/src/polus/plugins/_plugins/manifests/manifest_utils.py deleted file mode 100644 index f2e8dbcf4..000000000 --- a/src/polus/plugins/_plugins/manifests/manifest_utils.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Utilities for manifest parsing and validation.""" -import json -import logging -import pathlib -from typing import Optional -from typing import Union - -import github -import requests -import validators -from polus.plugins._plugins._compat import PYDANTIC_V2 -from polus.plugins._plugins.models import ComputeSchema -from polus.plugins._plugins.models import WIPPPluginManifest -from pydantic import ValidationError -from pydantic import errors -from tqdm import tqdm - -if not PYDANTIC_V2: - from polus.plugins._plugins.utils import cast_version - -logger = logging.getLogger("polus.plugins") - -# Fields that must be in a plugin manifest -REQUIRED_FIELDS = [ - "name", - "version", - "description", - "author", - "containerId", - "inputs", - "outputs", - "ui", -] - - -class InvalidManifestError(Exception): - """Raised when manifest has validation errors.""" - - -def is_valid_manifest(plugin: dict) -> bool: - """Validate basic attributes of a plugin manifest. - - Args: - plugin: A parsed plugin json file - - Returns: - True if the plugin has the minimal json fields - """ - fields = list(plugin.keys()) - - for field in REQUIRED_FIELDS: - if field not in fields: - msg = f"Missing json field, {field}, in plugin manifest." - logger.error(msg) - return False - return True - - -def _load_manifest(manifest: Union[str, dict, pathlib.Path]) -> dict: - """Return manifest as dict from str (url or path) or pathlib.Path.""" - if isinstance(manifest, dict): # is dict - return manifest - if isinstance(manifest, pathlib.Path): # is path - if manifest.suffix != ".json": - msg = "plugin manifest must be a json file with .json extension." - raise ValueError(msg) - - with manifest.open("r", encoding="utf-8") as manifest_json: - manifest_ = json.load(manifest_json) - elif isinstance(manifest, str): # is str - if validators.url(manifest): # is url - manifest_ = requests.get(manifest, timeout=10).json() - else: # could (and should) be path - try: - manifest_ = _load_manifest(pathlib.Path(manifest)) - except Exception as exc: # was not a Path? # noqa - msg = "invalid manifest" - raise ValueError(msg) from exc - else: # is not str, dict, or path - msg = f"invalid manifest type {type(manifest)}" - raise ValueError(msg) - return manifest_ - - -def validate_manifest( - manifest: Union[str, dict, pathlib.Path], -) -> Union[WIPPPluginManifest, ComputeSchema]: - """Validate a plugin manifest against schema.""" - manifest = _load_manifest(manifest) - if not PYDANTIC_V2: # Pydantic V1 - manifest["version"] = cast_version( - manifest["version"], - ) # cast version to semver object - if "name" in manifest: - name = manifest["name"] - else: - msg = f"{manifest} has no value for name" - raise InvalidManifestError(msg) - - if "pluginHardwareRequirements" in manifest: - # Parse the manifest - try: - plugin = ComputeSchema(**manifest) - except ValidationError as e: - msg = f"{name} does not conform to schema" - raise InvalidManifestError(msg) from e - except BaseException as e: - raise e - else: - # Parse the manifest - try: - plugin = WIPPPluginManifest(**manifest) - except ValidationError as e: - msg = f"{manifest['name']} does not conform to schema" - raise InvalidManifestError( - msg, - ) from e - except BaseException as e: - raise e - return plugin - - -def _scrape_manifests( - repo: Union[str, github.Repository.Repository], # type: ignore - gh: github.Github, - min_depth: int = 1, - max_depth: Optional[int] = None, - return_invalid: bool = False, -) -> Union[list, tuple[list, list]]: - if max_depth is None: - max_depth = min_depth - min_depth = 0 - - if not max_depth >= min_depth: - msg = "max_depth is smaller than min_depth" - raise ValueError(msg) - - if isinstance(repo, str): - repo = gh.get_repo(repo) - - contents = list(repo.get_contents("")) # type: ignore - next_contents: list = [] - valid_manifests: list = [] - invalid_manifests: list = [] - - for d in range(0, max_depth): - for content in tqdm(contents, desc=f"{repo.full_name}: {d}"): - if content.type == "dir": - next_contents.extend(repo.get_contents(content.path)) # type: ignore - elif content.name.endswith(".json") and d >= min_depth: - manifest = json.loads(content.decoded_content) - if is_valid_manifest(manifest): - valid_manifests.append(manifest) - else: - invalid_manifests.append(manifest) - - contents = next_contents.copy() - next_contents = [] - - if return_invalid: - return valid_manifests, invalid_manifests - return valid_manifests - - -def _error_log(val_err: ValidationError, manifest: dict, fct: str) -> None: - report = [] - - for error in val_err.args[0]: - if isinstance(error, list): - error = error[0] # noqa - - if isinstance(error, AssertionError): - msg = ( - f"The plugin ({manifest['name']}) " - "failed an assertion check: {err.args[0]}" - ) - report.append(msg) - logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 - elif isinstance(error.exc, errors.MissingError): - msg = ( - f"The plugin ({manifest['name']}) " - "is missing fields: {err.loc_tuple()}" - ) - report.append(msg) - logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 - elif errors.ExtraError: - if error.loc_tuple()[0] in ["inputs", "outputs", "ui"]: - manifest_ = manifest[error.loc_tuple()[0]][error.loc_tuple()[1]]["name"] - msg = ( - f"The plugin ({manifest['name']}) " - "had unexpected values in the " - f"{error.loc_tuple()[0]} " - f"({manifest_}): " - f"{error.exc.args[0][0].loc_tuple()}" - ) - report.append(msg) - else: - msg = ( - f"The plugin ({manifest['name']}) " - "had an error: {err.exc.args[0][0]}" - ) - report.append(msg) - logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 - else: - str_val_err = str(val_err).replace("\n", ", ").replace(" ", " ") - msg = ( - f"{fct}: Uncaught manifest error in ({manifest['name']}): " - f"{str_val_err}" - ) - logger.warning(msg) diff --git a/src/polus/plugins/_plugins/models/PolusComputeSchema.json b/src/polus/plugins/_plugins/models/PolusComputeSchema.json deleted file mode 100644 index d4875d54d..000000000 --- a/src/polus/plugins/_plugins/models/PolusComputeSchema.json +++ /dev/null @@ -1,499 +0,0 @@ -{ - "definitions": { - "PluginInputType": { - "title": "PluginInputType", - "description": "An enumeration.", - "enum": [ - "path", - "string", - "number", - "array", - "boolean" - ] - }, - "PluginInput": { - "title": "PluginInput", - "type": "object", - "properties": { - "format": { - "title": "Format", - "type": "string" - }, - "label": { - "title": "Label", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "required": { - "title": "Required", - "type": "boolean" - }, - "type": { - "$ref": "#/definitions/PluginInputType" - }, - "default": { - "title": "Default", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - }, - { - "type": "boolean" - } - ] - } - }, - "required": [ - "name", - "required", - "type" - ] - }, - "PluginOutputType": { - "title": "PluginOutputType", - "description": "An enumeration.", - "enum": [ - "path" - ], - "type": "string" - }, - "PluginOutput": { - "title": "PluginOutput", - "type": "object", - "properties": { - "format": { - "title": "Format", - "type": "string" - }, - "label": { - "title": "Label", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "type": { - "$ref": "#/definitions/PluginOutputType" - } - }, - "required": [ - "name", - "type" - ] - }, - "GpuVendor": { - "title": "GpuVendor", - "description": "An enumeration.", - "enum": [ - "none", - "amd", - "tpu", - "nvidia" - ], - "type": "string" - }, - "PluginHardwareRequirements": { - "title": "PluginHardwareRequirements", - "type": "object", - "properties": { - "coresMax": { - "title": "Coresmax", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "coresMin": { - "title": "Coresmin", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "cpuAVX": { - "title": "Cpuavx", - "type": "boolean" - }, - "cpuAVX2": { - "title": "Cpuavx2", - "type": "boolean" - }, - "cpuMin": { - "title": "Cpumin", - "type": "string" - }, - "gpu": { - "$ref": "#/definitions/GpuVendor" - }, - "gpuCount": { - "title": "Gpucount", - "type": "number" - }, - "gpuDriverVersion": { - "title": "Gpudriverversion", - "type": "string" - }, - "gpuType": { - "title": "Gputype", - "type": "string" - }, - "outDirMax": { - "title": "Outdirmax", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "outDirMin": { - "title": "Outdirmin", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "ramMax": { - "title": "Rammax", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "ramMin": { - "title": "Rammin", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "tmpDirMax": { - "title": "Tmpdirmax", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "tmpDirMin": { - "title": "Tmpdirmin", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - } - } - }, - "ThenEntry": { - "title": "ThenEntry", - "type": "object", - "properties": { - "action": { - "title": "Action", - "type": "string" - }, - "input": { - "title": "Input", - "type": "string" - }, - "value": { - "title": "Value", - "type": "string" - } - }, - "required": [ - "action", - "input", - "value" - ] - }, - "ConditionEntry": { - "title": "ConditionEntry", - "type": "object", - "properties": { - "expression": { - "title": "Expression", - "type": "string" - } - }, - "required": [ - "expression" - ] - }, - "Validator": { - "title": "Validator", - "type": "object", - "properties": { - "then": { - "title": "Then", - "type": "array", - "items": { - "$ref": "#/definitions/ThenEntry" - } - }, - "validator": { - "title": "Validator", - "type": "array", - "items": { - "$ref": "#/definitions/ConditionEntry" - } - } - } - }, - "PluginUIType": { - "title": "PluginUIType", - "description": "An enumeration.", - "enum": [ - "checkbox", - "color", - "date", - "email", - "number", - "password", - "radio", - "range", - "text", - "time" - ] - }, - "PluginUIInput": { - "title": "PluginUIInput", - "type": "object", - "properties": { - "bind": { - "title": "Bind", - "type": "string" - }, - "condition": { - "title": "Condition", - "anyOf": [ - { - "type": "array", - "items": { - "$ref": "#/definitions/Validator" - } - }, - { - "type": "string" - } - ] - }, - "default": { - "title": "Default", - "anyOf": [ - { - "type": "string" - }, - { - "type": "number" - }, - { - "type": "boolean" - } - ] - }, - "description": { - "title": "Description", - "type": "string" - }, - "fieldset": { - "title": "Fieldset", - "type": "array", - "items": { - "type": "string" - } - }, - "hidden": { - "title": "Hidden", - "type": "boolean" - }, - "key": { - "title": "Key", - "type": "string" - }, - "title": { - "title": "Title", - "type": "string" - }, - "type": { - "$ref": "#/definitions/PluginUIType" - } - }, - "required": [ - "key", - "title", - "type" - ] - }, - "PluginUIOutput": { - "title": "PluginUIOutput", - "type": "object", - "properties": { - "description": { - "title": "Description", - "type": "string" - }, - "format": { - "title": "Format", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "type": { - "$ref": "#/definitions/PluginUIType" - }, - "website": { - "title": "Website", - "type": "string" - } - }, - "required": [ - "description", - "name", - "type" - ] - }, - "PluginSchema": { - "title": "PluginSchema", - "type": "object", - "properties": { - "author": { - "title": "Author", - "type": "string" - }, - "baseCommand": { - "title": "Basecommand", - "type": "array", - "items": { - "type": "string" - } - }, - "citation": { - "title": "Citation", - "type": "string" - }, - "containerId": { - "title": "Containerid", - "type": "string" - }, - "customInputs": { - "title": "Custominputs", - "type": "boolean" - }, - "description": { - "title": "Description", - "type": "string" - }, - "inputs": { - "title": "Inputs", - "type": "array", - "items": { - "$ref": "#/definitions/PluginInput" - } - }, - "institution": { - "title": "Institution", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "outputs": { - "title": "Outputs", - "type": "array", - "items": { - "$ref": "#/definitions/PluginOutput" - } - }, - "pluginHardwareRequirements": { - "$ref": "#/definitions/PluginHardwareRequirements" - }, - "repository": { - "title": "Repository", - "type": "string" - }, - "title": { - "title": "Title", - "type": "string" - }, - "ui": { - "title": "Ui", - "type": "array", - "items": { - "anyOf": [ - { - "$ref": "#/definitions/PluginUIInput" - }, - { - "$ref": "#/definitions/PluginUIOutput" - } - ] - } - }, - "version": { - "title": "Version", - "examples": [ - "0.1.0", - "0.1.0rc1" - ], - "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", - "type": "string" - }, - "website": { - "title": "Website", - "type": "string" - } - }, - "required": [ - "containerId", - "description", - "inputs", - "name", - "outputs", - "pluginHardwareRequirements", - "title", - "ui", - "version" - ] - } - } -} diff --git a/src/polus/plugins/_plugins/models/PolusComputeSchema.ts b/src/polus/plugins/_plugins/models/PolusComputeSchema.ts deleted file mode 100644 index 184ebbf91..000000000 --- a/src/polus/plugins/_plugins/models/PolusComputeSchema.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** -/* This file was automatically generated from pydantic models by running pydantic2ts. -/* Do not modify it by hand - just update the pydantic models and then re-run the script -*/ - -export type GpuVendor = "none" | "amd" | "tpu" | "nvidia"; -export type PluginInputType = "path" | "string" | "number" | "array" | "boolean"; -export type PluginOutputType = "path"; -export type PluginUIType = - | "checkbox" - | "color" - | "date" - | "email" - | "number" - | "password" - | "radio" - | "range" - | "text" - | "time"; - -export interface ConditionEntry { - expression: string; -} -export interface Model {} -export interface PluginHardwareRequirements { - coresMax?: string | number; - coresMin?: string | number; - cpuAVX?: boolean; - cpuAVX2?: boolean; - cpuMin?: string; - gpu?: GpuVendor; - gpuCount?: number; - gpuDriverVersion?: string; - gpuType?: string; - outDirMax?: string | number; - outDirMin?: string | number; - ramMax?: string | number; - ramMin?: string | number; - tmpDirMax?: string | number; - tmpDirMin?: string | number; -} -export interface PluginInput { - format?: string; - label?: string; - name: string; - required: boolean; - type: PluginInputType; - default?: string | number | boolean; -} -export interface PluginOutput { - format?: string; - label?: string; - name: string; - type: PluginOutputType; -} -export interface PluginSchema { - author?: string; - baseCommand?: string[]; - citation?: string; - containerId: string; - customInputs?: boolean; - description: string; - inputs: PluginInput[]; - institution?: string; - name: string; - outputs: PluginOutput[]; - pluginHardwareRequirements: PluginHardwareRequirements; - repository?: string; - title: string; - ui: (PluginUIInput | PluginUIOutput)[]; - version: string; - website?: string; -} -export interface PluginUIInput { - bind?: string; - condition?: Validator[] | string; - default?: string | number | boolean; - description?: string; - fieldset?: string[]; - hidden?: boolean; - key: string; - title: string; - type: PluginUIType; -} -export interface Validator { - then?: ThenEntry[]; - validator?: ConditionEntry[]; -} -export interface ThenEntry { - action: string; - input: string; - value: string; -} -export interface PluginUIOutput { - description: string; - format?: string; - name: string; - type: PluginUIType; - website?: string; -} diff --git a/src/polus/plugins/_plugins/models/__init__.py b/src/polus/plugins/_plugins/models/__init__.py deleted file mode 100644 index 9b9092c75..000000000 --- a/src/polus/plugins/_plugins/models/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Pydantic Models based on JSON schemas.""" - -import pydantic - -PYDANTIC_VERSION = pydantic.__version__ - -if PYDANTIC_VERSION.split(".")[0] == "1": - from polus.plugins._plugins.models.pydanticv1.compute import ( - PluginSchema as ComputeSchema, - ) - from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import ( - PluginUIInput, - ) - from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import ( - PluginUIOutput, - ) - from polus.plugins._plugins.models.pydanticv1.wipp import WIPPPluginManifest -elif PYDANTIC_VERSION.split(".")[0] == "2": - from polus.plugins._plugins.models.pydanticv2.compute import ( - PluginSchema as ComputeSchema, - ) - from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import ( - PluginUIInput, - ) - from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import ( - PluginUIOutput, - ) - from polus.plugins._plugins.models.pydanticv2.wipp import WIPPPluginManifest - -__all__ = [ - "WIPPPluginManifest", - "PluginUIInput", - "PluginUIOutput", - "ComputeSchema", -] diff --git a/src/polus/plugins/_plugins/models/pydanticv1/PolusComputeSchema.py b/src/polus/plugins/_plugins/models/pydanticv1/PolusComputeSchema.py deleted file mode 100644 index a40b5b402..000000000 --- a/src/polus/plugins/_plugins/models/pydanticv1/PolusComputeSchema.py +++ /dev/null @@ -1,137 +0,0 @@ -# generated by datamodel-codegen: -# timestamp: 2022-09-21T03:41:58+00:00 - -from __future__ import annotations - -from enum import Enum -from typing import Any - -from pydantic import BaseModel -from pydantic import Field -from pydantic import constr - - -class Model(BaseModel): - __root__: Any - - -class PluginInputType(Enum): - path = "path" - string = "string" - number = "number" - array = "array" - boolean = "boolean" - - -class PluginInput(BaseModel): - format: str | None = Field(None, title="Format") - label: str | None = Field(None, title="Label") - name: str = Field(..., title="Name") - required: bool = Field(..., title="Required") - type: PluginInputType - default: str | float | bool | None = Field(None, title="Default") - - -class PluginOutputType(Enum): - path = "path" - - -class PluginOutput(BaseModel): - format: str | None = Field(None, title="Format") - label: str | None = Field(None, title="Label") - name: str = Field(..., title="Name") - type: PluginOutputType - - -class GpuVendor(Enum): - none = "none" - amd = "amd" - tpu = "tpu" - nvidia = "nvidia" - - -class PluginHardwareRequirements(BaseModel): - coresMax: str | float | None = Field(None, title="Coresmax") - coresMin: str | float | None = Field(None, title="Coresmin") - cpuAVX: bool | None = Field(None, title="Cpuavx") - cpuAVX2: bool | None = Field(None, title="Cpuavx2") - cpuMin: str | None = Field(None, title="Cpumin") - gpu: GpuVendor | None = None - gpuCount: float | None = Field(None, title="Gpucount") - gpuDriverVersion: str | None = Field(None, title="Gpudriverversion") - gpuType: str | None = Field(None, title="Gputype") - outDirMax: str | float | None = Field(None, title="Outdirmax") - outDirMin: str | float | None = Field(None, title="Outdirmin") - ramMax: str | float | None = Field(None, title="Rammax") - ramMin: str | float | None = Field(None, title="Rammin") - tmpDirMax: str | float | None = Field(None, title="Tmpdirmax") - tmpDirMin: str | float | None = Field(None, title="Tmpdirmin") - - -class ThenEntry(BaseModel): - action: str = Field(..., title="Action") - input: str = Field(..., title="Input") - value: str = Field(..., title="Value") - - -class ConditionEntry(BaseModel): - expression: str = Field(..., title="Expression") - - -class Validator(BaseModel): - then: list[ThenEntry] | None = Field(None, title="Then") - validator: list[ConditionEntry] | None = Field(None, title="Validator") - - -class PluginUIType(Enum): - checkbox = "checkbox" - color = "color" - date = "date" - email = "email" - number = "number" - password = "password" - radio = "radio" - range = "range" - text = "text" - time = "time" - - -class PluginUIInput(BaseModel): - bind: str | None = Field(None, title="Bind") - condition: list[Validator] | str | None = Field(None, title="Condition") - default: str | float | bool | None = Field(None, title="Default") - description: str | None = Field(None, title="Description") - fieldset: list[str] | None = Field(None, title="Fieldset") - hidden: bool | None = Field(None, title="Hidden") - key: str = Field(..., title="Key") - title: str = Field(..., title="Title") - type: PluginUIType - - -class PluginUIOutput(BaseModel): - description: str = Field(..., title="Description") - format: str | None = Field(None, title="Format") - name: str = Field(..., title="Name") - type: PluginUIType - website: str | None = Field(None, title="Website") - - -class PluginSchema(BaseModel): - author: str | None = Field(None, title="Author") - baseCommand: list[str] | None = Field(None, title="Basecommand") - citation: str | None = Field(None, title="Citation") - containerId: str = Field(..., title="Containerid") - customInputs: bool | None = Field(None, title="Custominputs") - description: str = Field(..., title="Description") - inputs: list[PluginInput] = Field(..., title="Inputs") - institution: str | None = Field(None, title="Institution") - name: str = Field(..., title="Name") - outputs: list[PluginOutput] = Field(..., title="Outputs") - pluginHardwareRequirements: PluginHardwareRequirements - repository: str | None = Field(None, title="Repository") - title: str = Field(..., title="Title") - ui: list[PluginUIInput | PluginUIOutput] = Field(..., title="Ui") - version: constr( - regex=r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", - ) = Field(..., examples=["0.1.0", "0.1.0rc1"], title="Version") - website: str | None = Field(None, title="Website") diff --git a/src/polus/plugins/_plugins/models/pydanticv1/WIPPPluginSchema.py b/src/polus/plugins/_plugins/models/pydanticv1/WIPPPluginSchema.py deleted file mode 100644 index 718d3a3fa..000000000 --- a/src/polus/plugins/_plugins/models/pydanticv1/WIPPPluginSchema.py +++ /dev/null @@ -1,233 +0,0 @@ -# generated by datamodel-codegen: -# timestamp: 2023-01-04T14:54:38+00:00 - -from __future__ import annotations - -from enum import Enum -from typing import Any - -from pydantic import AnyUrl -from pydantic import BaseModel -from pydantic import Field -from pydantic import constr - - -class Type(Enum): - collection = "collection" - stitchingVector = "stitchingVector" - tensorflowModel = "tensorflowModel" - csvCollection = "csvCollection" - pyramid = "pyramid" - pyramidAnnotation = "pyramidAnnotation" - notebook = "notebook" - genericData = "genericData" - string = "string" - number = "number" - integer = "integer" - enum = "enum" - array = "array" - boolean = "boolean" - - -class Input(BaseModel): - name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( - ..., - description="Input name as expected by the plugin CLI", - examples=["inputImages", "fileNamePattern", "thresholdValue"], - title="Input name", - ) - type: Type = Field( - ..., - examples=["collection", "string", "number"], - title="Input Type", - ) - description: constr(regex=r"^(.*)$") = Field( - ..., - examples=["Input Images"], - title="Input description", - ) - required: bool | None = Field( - True, - description="Whether an input is required or not", - examples=[True], - title="Required input", - ) - - -class Type1(Enum): - collection = "collection" - stitchingVector = "stitchingVector" - tensorflowModel = "tensorflowModel" - tensorboardLogs = "tensorboardLogs" - csvCollection = "csvCollection" - pyramid = "pyramid" - pyramidAnnotation = "pyramidAnnotation" - genericData = "genericData" - - -class Output(BaseModel): - name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( - ..., - examples=["outputCollection"], - title="Output name", - ) - type: Type1 = Field( - ..., - examples=["stitchingVector", "collection"], - title="Output type", - ) - description: constr(regex=r"^(.*)$") = Field( - ..., - examples=["Output collection"], - title="Output description", - ) - - -class UiItem(BaseModel): - key: Any | Any = Field( - ..., - description="Key of the input which this UI definition applies to, the expected format is 'inputs.inputName'. Special keyword 'fieldsets' can be used to define arrangement of inputs by sections.", - examples=["inputs.inputImages", "inputs.fileNamPattern", "fieldsets"], - title="UI key", - ) - - -class CudaRequirements(BaseModel): - deviceMemoryMin: float | None = Field( - 0, - examples=[100], - title="Minimum device memory", - ) - cudaComputeCapability: str | list[Any] | None = Field( - None, - description="Specify either a single minimum value, or an array of valid values", - examples=["8.0", ["3.5", "5.0", "6.0", "7.0", "7.5", "8.0"]], - title="The cudaComputeCapability Schema", - ) - - -class ResourceRequirements(BaseModel): - ramMin: float | None = Field( - None, - examples=[2048], - title="Minimum RAM in mebibytes (Mi)", - ) - coresMin: float | None = Field( - None, - examples=[1], - title="Minimum number of CPU cores", - ) - cpuAVX: bool | None = Field( - False, - examples=[True], - title="Advanced Vector Extensions (AVX) CPU capability required", - ) - cpuAVX2: bool | None = Field( - False, - examples=[False], - title="Advanced Vector Extensions 2 (AVX2) CPU capability required", - ) - gpu: bool | None = Field( - False, - examples=[True], - title="GPU/accelerator required", - ) - cudaRequirements: CudaRequirements | None = Field( - {}, - examples=[{"deviceMemoryMin": 100, "cudaComputeCapability": "8.0"}], - title="GPU Cuda-related requirements", - ) - - -class WippPluginManifest(BaseModel): - name: constr(regex=r"^(.*)$", min_length=1) = Field( - ..., - description="Name of the plugin (format: org/name)", - examples=["wipp/plugin-example"], - title="Plugin name", - ) - version: constr(regex=r"^(.*)$", min_length=1) = Field( - ..., - description="Version of the plugin (semantic versioning preferred)", - examples=["1.0.0"], - title="Plugin version", - ) - title: constr(regex=r"^(.*)$", min_length=1) = Field( - ..., - description="Plugin title to display in WIPP forms", - examples=["WIPP Plugin example"], - title="Plugin title", - ) - description: constr(regex=r"^(.*)$", min_length=1) = Field( - ..., - examples=["Custom image segmentation plugin"], - title="Short description of the plugin", - ) - author: constr(regex="^(.*)$") | None | None = Field( - "", - examples=["FirstName LastName"], - title="Author(s)", - ) - institution: constr(regex="^(.*)$") | None | None = Field( - "", - examples=["National Institute of Standards and Technology"], - title="Institution", - ) - repository: AnyUrl | None | None = Field( - "", - examples=["https://github.com/usnistgov/WIPP"], - title="Source code repository", - ) - website: AnyUrl | None | None = Field( - "", - examples=["http://usnistgov.github.io/WIPP"], - title="Website", - ) - citation: constr(regex="^(.*)$") | None | None = Field( - "", - examples=[ - "Peter Bajcsy, Joe Chalfoun, and Mylene Simon (2018). Web Microanalysis of Big Image Data. Springer-Verlag International", - ], - title="Citation", - ) - containerId: constr(regex=r"^(.*)$") = Field( - ..., - description="Docker image ID", - examples=["docker.io/wipp/plugin-example:1.0.0"], - title="ContainerId", - ) - baseCommand: list[str] | None = Field( - None, - description="Base command to use while running container image", - examples=[["python3", "/opt/executable/main.py"]], - title="Base command", - ) - inputs: list[Input] = Field( - ..., - description="Defines inputs to the plugin", - title="List of Inputs", - unique_items=True, - ) - outputs: list[Output] = Field( - ..., - description="Defines the outputs of the plugin", - title="List of Outputs", - ) - ui: list[UiItem] = Field(..., title="Plugin form UI definition") - resourceRequirements: ResourceRequirements | None = Field( - {}, - examples=[ - { - "ramMin": 2048, - "coresMin": 1, - "cpuAVX": True, - "cpuAVX2": False, - "gpu": True, - "cudaRequirements": { - "deviceMemoryMin": 100, - "cudaComputeCapability": "8.0", - }, - }, - ], - title="Plugin Resource Requirements", - ) diff --git a/src/polus/plugins/_plugins/models/pydanticv1/__init__.py b/src/polus/plugins/_plugins/models/pydanticv1/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/polus/plugins/_plugins/models/pydanticv1/compute.py b/src/polus/plugins/_plugins/models/pydanticv1/compute.py deleted file mode 100644 index 2a34a3ae7..000000000 --- a/src/polus/plugins/_plugins/models/pydanticv1/compute.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Extending automatically generated compute model. - -This file modifies and extend certain fields and -functions of PolusComputeSchema.py which is automatically -generated by datamodel-codegen from JSON schema. -""" - -from polus.plugins._plugins.io import IOBase -from polus.plugins._plugins.io import Version -from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import PluginInput -from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import PluginOutput -from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import PluginSchema - - -class PluginInput(PluginInput, IOBase): # type: ignore - """Base Class for Input Args.""" - - -class PluginOutput(PluginOutput, IOBase): # type: ignore - """Base Class for Output Args.""" - - -class PluginSchema(PluginSchema): # type: ignore - """Extended Compute Plugin Schema with extended IO defs.""" - - inputs: list[PluginInput] - outputs: list[PluginOutput] - version: Version diff --git a/src/polus/plugins/_plugins/models/pydanticv1/wipp.py b/src/polus/plugins/_plugins/models/pydanticv1/wipp.py deleted file mode 100644 index 6557355a2..000000000 --- a/src/polus/plugins/_plugins/models/pydanticv1/wipp.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Extending automatically generated wipp model. - -This file modifies and extend certain fields and -functions of WIPPPluginSchema.py which is automatically -generated by datamodel-codegen from JSON schema. -""" -from typing import Literal -from typing import Optional -from typing import Union - -from polus.plugins._plugins.io import Input -from polus.plugins._plugins.io import Output -from polus.plugins._plugins.io import Version -from polus.plugins._plugins.models.pydanticv1.WIPPPluginSchema import ( - ResourceRequirements, -) -from polus.plugins._plugins.models.pydanticv1.WIPPPluginSchema import WippPluginManifest -from pydantic import BaseModel -from pydantic import Field - - -class UI1(BaseModel): - """Base class for UI items.""" - - key: str = Field(constr=r"^inputs.[a-zA-Z0-9][-a-zA-Z0-9]*$") - title: str - description: Optional[str] - condition: Optional[str] - default: Optional[Union[str, float, int, bool]] - hidden: Optional[bool] = Field(default=False) - bind: Optional[str] - - -class FieldSet(BaseModel): - """Base class for FieldSet.""" - - title: str - fields: list[str] = Field(min_items=1, unique_items=True) - - -class UI2(BaseModel): - """UI items class for fieldsets.""" - - key: Literal["fieldsets"] - fieldsets: list[FieldSet] = Field(min_items=1, unique_items=True) - - -class WIPPPluginManifest(WippPluginManifest): - """Extended WIPP Plugin Schema.""" - - inputs: list[Input] = Field( - ..., - description="Defines inputs to the plugin", - title="List of Inputs", - ) - outputs: list[Output] = Field( - ..., - description="Defines the outputs of the plugin", - title="List of Outputs", - ) - ui: list[Union[UI1, UI2]] = Field(..., title="Plugin form UI definition") - version: Version - resourceRequirements: Optional[ResourceRequirements] = Field( # noqa - None, - examples=[ - { - "ramMin": 2048, - "coresMin": 1, - "cpuAVX": True, - "cpuAVX2": False, - "gpu": True, - "cudaRequirements": { - "deviceMemoryMin": 100, - "cudaComputeCapability": "8.0", - }, - }, - ], - title="Plugin Resource Requirements", - ) diff --git a/src/polus/plugins/_plugins/models/pydanticv2/PolusComputeSchema.py b/src/polus/plugins/_plugins/models/pydanticv2/PolusComputeSchema.py deleted file mode 100644 index d87a986fd..000000000 --- a/src/polus/plugins/_plugins/models/pydanticv2/PolusComputeSchema.py +++ /dev/null @@ -1,136 +0,0 @@ -# generated by datamodel-codegen: edited by Camilo Velez -# timestamp: 2022-09-21T03:41:58+00:00 - -from __future__ import annotations - -from enum import Enum -from typing import Annotated - -from pydantic import BaseModel -from pydantic import Field -from pydantic import StringConstraints - - -class PluginInputType(Enum): - path = "path" - string = "string" - number = "number" - array = "array" - boolean = "boolean" - - -class PluginInput(BaseModel): - format: str | None = Field(None, title="Format") - label: str | None = Field(None, title="Label") - name: str = Field(..., title="Name") - required: bool = Field(..., title="Required") - type: PluginInputType - default: str | float | bool | None = Field(None, title="Default") - - -class PluginOutputType(Enum): - path = "path" - - -class PluginOutput(BaseModel): - format: str | None = Field(None, title="Format") - label: str | None = Field(None, title="Label") - name: str = Field(..., title="Name") - type: PluginOutputType - - -class GpuVendor(Enum): - none = "none" - amd = "amd" - tpu = "tpu" - nvidia = "nvidia" - - -class PluginHardwareRequirements(BaseModel): - coresMax: str | float | None = Field(None, title="Coresmax") - coresMin: str | float | None = Field(None, title="Coresmin") - cpuAVX: bool | None = Field(None, title="Cpuavx") - cpuAVX2: bool | None = Field(None, title="Cpuavx2") - cpuMin: str | None = Field(None, title="Cpumin") - gpu: GpuVendor | None = None - gpuCount: float | None = Field(None, title="Gpucount") - gpuDriverVersion: str | None = Field(None, title="Gpudriverversion") - gpuType: str | None = Field(None, title="Gputype") - outDirMax: str | float | None = Field(None, title="Outdirmax") - outDirMin: str | float | None = Field(None, title="Outdirmin") - ramMax: str | float | None = Field(None, title="Rammax") - ramMin: str | float | None = Field(None, title="Rammin") - tmpDirMax: str | float | None = Field(None, title="Tmpdirmax") - tmpDirMin: str | float | None = Field(None, title="Tmpdirmin") - - -class ThenEntry(BaseModel): - action: str = Field(..., title="Action") - input: str = Field(..., title="Input") - value: str = Field(..., title="Value") - - -class ConditionEntry(BaseModel): - expression: str = Field(..., title="Expression") - - -class Validator(BaseModel): - then: list[ThenEntry] | None = Field(None, title="Then") - validator: list[ConditionEntry] | None = Field(None, title="Validator") - - -class PluginUIType(Enum): - checkbox = "checkbox" - color = "color" - date = "date" - email = "email" - number = "number" - password = "password" - radio = "radio" - range = "range" - text = "text" - time = "time" - - -class PluginUIInput(BaseModel): - bind: str | None = Field(None, title="Bind") - condition: list[Validator] | str | None = Field(None, title="Condition") - default: str | float | bool | None = Field(None, title="Default") - description: str | None = Field(None, title="Description") - fieldset: list[str] | None = Field(None, title="Fieldset") - hidden: bool | None = Field(None, title="Hidden") - key: str = Field(..., title="Key") - title: str = Field(..., title="Title") - type: PluginUIType - - -class PluginUIOutput(BaseModel): - description: str = Field(..., title="Description") - format: str | None = Field(None, title="Format") - name: str = Field(..., title="Name") - type: PluginUIType - website: str | None = Field(None, title="Website") - - -class PluginSchema(BaseModel): - author: str | None = Field(None, title="Author") - baseCommand: list[str] | None = Field(None, title="Basecommand") - citation: str | None = Field(None, title="Citation") - containerId: str = Field(..., title="Containerid") - customInputs: bool | None = Field(None, title="Custominputs") - description: str = Field(..., title="Description") - inputs: list[PluginInput] = Field(..., title="Inputs") - institution: str | None = Field(None, title="Institution") - name: str = Field(..., title="Name") - outputs: list[PluginOutput] = Field(..., title="Outputs") - pluginHardwareRequirements: PluginHardwareRequirements - repository: str | None = Field(None, title="Repository") - title: str = Field(..., title="Title") - ui: list[PluginUIInput | PluginUIOutput] = Field(..., title="Ui") - version: Annotated[ - str, - StringConstraints( - pattern=r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", - ), - ] = Field(..., examples=["0.1.0", "0.1.0rc1"], title="Version") - website: str | None = Field(None, title="Website") diff --git a/src/polus/plugins/_plugins/models/pydanticv2/WIPPPluginSchema.py b/src/polus/plugins/_plugins/models/pydanticv2/WIPPPluginSchema.py deleted file mode 100644 index 099cb32d2..000000000 --- a/src/polus/plugins/_plugins/models/pydanticv2/WIPPPluginSchema.py +++ /dev/null @@ -1,241 +0,0 @@ -# generated by datamodel-codegen: edited by Camilo Velez -# timestamp: 2023-01-04T14:54:38+00:00 - -from __future__ import annotations - -from enum import Enum -from typing import Annotated -from typing import Any - -from pydantic import AnyUrl -from pydantic import BaseModel -from pydantic import Field -from pydantic import StringConstraints - - -class Type(Enum): - collection = "collection" - stitchingVector = "stitchingVector" - tensorflowModel = "tensorflowModel" - csvCollection = "csvCollection" - pyramid = "pyramid" - pyramidAnnotation = "pyramidAnnotation" - notebook = "notebook" - genericData = "genericData" - string = "string" - number = "number" - integer = "integer" - enum = "enum" - array = "array" - boolean = "boolean" - - -class Input(BaseModel): - name: Annotated[ - str, - StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), - ] = Field( - ..., - description="Input name as expected by the plugin CLI", - examples=["inputImages", "fileNamePattern", "thresholdValue"], - title="Input name", - ) - type: Type = Field( - ..., - examples=["collection", "string", "number"], - title="Input Type", - ) - description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( - ..., - examples=["Input Images"], - title="Input description", - ) - required: bool | None = Field( - True, - description="Whether an input is required or not", - examples=[True], - title="Required input", - ) - - -class Type1(Enum): - collection = "collection" - stitchingVector = "stitchingVector" - tensorflowModel = "tensorflowModel" - tensorboardLogs = "tensorboardLogs" - csvCollection = "csvCollection" - pyramid = "pyramid" - pyramidAnnotation = "pyramidAnnotation" - genericData = "genericData" - - -class Output(BaseModel): - name: Annotated[ - str, - StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), - ] = Field(..., examples=["outputCollection"], title="Output name") - type: Type1 = Field( - ..., - examples=["stitchingVector", "collection"], - title="Output type", - ) - description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( - ..., - examples=["Output collection"], - title="Output description", - ) - - -class UiItem(BaseModel): - key: Any | Any = Field( - ..., - description="Key of the input which this UI definition applies to, the expected format is 'inputs.inputName'. Special keyword 'fieldsets' can be used to define arrangement of inputs by sections.", - examples=["inputs.inputImages", "inputs.fileNamPattern", "fieldsets"], - title="UI key", - ) - - -class CudaRequirements(BaseModel): - deviceMemoryMin: float | None = Field( - 0, - examples=[100], - title="Minimum device memory", - ) - cudaComputeCapability: str | list[Any] | None = Field( - None, - description="Specify either a single minimum value, or an array of valid values", - examples=["8.0", ["3.5", "5.0", "6.0", "7.0", "7.5", "8.0"]], - title="The cudaComputeCapability Schema", - ) - - -class ResourceRequirements(BaseModel): - ramMin: float | None = Field( - None, - examples=[2048], - title="Minimum RAM in mebibytes (Mi)", - ) - coresMin: float | None = Field( - None, - examples=[1], - title="Minimum number of CPU cores", - ) - cpuAVX: bool | None = Field( - False, - examples=[True], - title="Advanced Vector Extensions (AVX) CPU capability required", - ) - cpuAVX2: bool | None = Field( - False, - examples=[False], - title="Advanced Vector Extensions 2 (AVX2) CPU capability required", - ) - gpu: bool | None = Field( - False, - examples=[True], - title="GPU/accelerator required", - ) - cudaRequirements: CudaRequirements | None = Field( - {}, - examples=[{"deviceMemoryMin": 100, "cudaComputeCapability": "8.0"}], - title="GPU Cuda-related requirements", - ) - - -class WippPluginManifest(BaseModel): - name: Annotated[str, StringConstraints(pattern=r"^(.*)$", min_length=1)] = Field( - ..., - description="Name of the plugin (format: org/name)", - examples=["wipp/plugin-example"], - title="Plugin name", - ) - version: Annotated[str, StringConstraints(pattern=r"^(.*)$", min_length=1)] = Field( - ..., - description="Version of the plugin (semantic versioning preferred)", - examples=["1.0.0"], - title="Plugin version", - ) - title: Annotated[str, StringConstraints(pattern=r"^(.*)$", min_length=1)] = Field( - ..., - description="Plugin title to display in WIPP forms", - examples=["WIPP Plugin example"], - title="Plugin title", - ) - description: Annotated[ - str, - StringConstraints(pattern=r"^(.*)$", min_length=1), - ] = Field( - ..., - examples=["Custom image segmentation plugin"], - title="Short description of the plugin", - ) - author: Annotated[str, StringConstraints(pattern="^(.*)$")] | None | None = Field( - "", - examples=["FirstName LastName"], - title="Author(s)", - ) - institution: Annotated[ - str, - StringConstraints(pattern="^(.*)$"), - ] | None | None = Field( - "", - examples=["National Institute of Standards and Technology"], - title="Institution", - ) - repository: AnyUrl | None | None = Field( - "", - examples=["https://github.com/usnistgov/WIPP"], - title="Source code repository", - ) - website: AnyUrl | None | None = Field( - "", - examples=["http://usnistgov.github.io/WIPP"], - title="Website", - ) - citation: Annotated[str, StringConstraints(pattern="^(.*)$")] | None | None = Field( - "", - examples=[ - "Peter Bajcsy, Joe Chalfoun, and Mylene Simon (2018). Web Microanalysis of Big Image Data. Springer-Verlag International", - ], - title="Citation", - ) - containerId: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( - ..., - description="Docker image ID", - examples=["docker.io/wipp/plugin-example:1.0.0"], - title="ContainerId", - ) - baseCommand: list[str] | None = Field( - None, - description="Base command to use while running container image", - examples=[["python3", "/opt/executable/main.py"]], - title="Base command", - ) - inputs: set[Input] = Field( - ..., - description="Defines inputs to the plugin", - title="List of Inputs", - ) - outputs: list[Output] = Field( - ..., - description="Defines the outputs of the plugin", - title="List of Outputs", - ) - ui: list[UiItem] = Field(..., title="Plugin form UI definition") - resourceRequirements: ResourceRequirements | None = Field( - {}, - examples=[ - { - "ramMin": 2048, - "coresMin": 1, - "cpuAVX": True, - "cpuAVX2": False, - "gpu": True, - "cudaRequirements": { - "deviceMemoryMin": 100, - "cudaComputeCapability": "8.0", - }, - }, - ], - title="Plugin Resource Requirements", - ) diff --git a/src/polus/plugins/_plugins/models/pydanticv2/__init__.py b/src/polus/plugins/_plugins/models/pydanticv2/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/polus/plugins/_plugins/models/pydanticv2/compute.py b/src/polus/plugins/_plugins/models/pydanticv2/compute.py deleted file mode 100644 index 250574f12..000000000 --- a/src/polus/plugins/_plugins/models/pydanticv2/compute.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Extending automatically generated compute model. - -This file modifies and extend certain fields and -functions of PolusComputeSchema.py which is automatically -generated by datamodel-codegen from JSON schema. -""" - -from polus.plugins._plugins.io import IOBase -from polus.plugins._plugins.io import Version -from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import PluginInput -from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import PluginOutput -from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import PluginSchema - - -class PluginInput(PluginInput, IOBase): # type: ignore - """Base Class for Input Args.""" - - -class PluginOutput(PluginOutput, IOBase): # type: ignore - """Base Class for Output Args.""" - - -class PluginSchema(PluginSchema): # type: ignore - """Extended Compute Plugin Schema with extended IO defs.""" - - inputs: list[PluginInput] - outputs: list[PluginOutput] - version: Version diff --git a/src/polus/plugins/_plugins/models/pydanticv2/wipp.py b/src/polus/plugins/_plugins/models/pydanticv2/wipp.py deleted file mode 100644 index 41c0dff31..000000000 --- a/src/polus/plugins/_plugins/models/pydanticv2/wipp.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Extending automatically generated wipp model. - -This file modifies and extend certain fields and -functions of WIPPPluginSchema.py which is automatically -generated by datamodel-codegen from JSON schema. -""" -from typing import Literal -from typing import Optional -from typing import Union - -from polus.plugins._plugins.io import Input -from polus.plugins._plugins.io import Output -from polus.plugins._plugins.io import Version -from polus.plugins._plugins.models.pydanticv2.WIPPPluginSchema import ( - ResourceRequirements, -) -from polus.plugins._plugins.models.pydanticv2.WIPPPluginSchema import WippPluginManifest -from pydantic import BaseModel -from pydantic import Field - - -class UI1(BaseModel): - """Base class for UI items.""" - - key: str = Field(constr=r"^inputs.[a-zA-Z0-9][-a-zA-Z0-9]*$") - title: str - description: Optional[str] = None - condition: Optional[str] = None - default: Optional[Union[str, float, int, bool]] = None - hidden: Optional[bool] = Field(default=False) - bind: Optional[str] = None - - -class FieldSet(BaseModel): - """Base class for FieldSet.""" - - title: str - fields: set[str] = Field(min_length=1) - - -class UI2(BaseModel): - """UI items class for fieldsets.""" - - key: Literal["fieldsets"] - fieldsets: set[FieldSet] = Field(min_length=1) - - -class WIPPPluginManifest(WippPluginManifest): - """Extended WIPP Plugin Schema.""" - - inputs: list[Input] = Field( - ..., - description="Defines inputs to the plugin", - title="List of Inputs", - ) - outputs: list[Output] = Field( - ..., - description="Defines the outputs of the plugin", - title="List of Outputs", - ) - ui: list[Union[UI1, UI2]] = Field(..., title="Plugin form UI definition") - version: Version - resourceRequirements: Optional[ResourceRequirements] = Field( # noqa - None, - examples=[ - { - "ramMin": 2048, - "coresMin": 1, - "cpuAVX": True, - "cpuAVX2": False, - "gpu": True, - "cudaRequirements": { - "deviceMemoryMin": 100, - "cudaComputeCapability": "8.0", - }, - }, - ], - title="Plugin Resource Requirements", - ) diff --git a/src/polus/plugins/_plugins/models/wipp-plugin-manifest-schema.json b/src/polus/plugins/_plugins/models/wipp-plugin-manifest-schema.json deleted file mode 100644 index 8a407aecd..000000000 --- a/src/polus/plugins/_plugins/models/wipp-plugin-manifest-schema.json +++ /dev/null @@ -1,726 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://raw.githubusercontent.com/usnistgov/WIPP-Plugins-base-templates/master/plugin-manifest/schema/wipp-plugin-manifest-schema.json", - "type": "object", - "title": "WIPP Plugin manifest", - "default": null, - "required": [ - "name", - "version", - "title", - "description", - "containerId", - "inputs", - "outputs", - "ui" - ], - "properties": { - "name": { - "$id": "#/properties/name", - "type": "string", - "title": "Plugin name", - "description": "Name of the plugin (format: org/name)", - "default": "", - "examples": [ - "wipp/plugin-example" - ], - "minLength": 1, - "pattern": "^(.*)$" - }, - "version": { - "$id": "#/properties/version", - "type": "string", - "title": "Plugin version", - "description": "Version of the plugin (semantic versioning preferred)", - "default": "", - "examples": [ - "1.0.0" - ], - "minLength": 1, - "pattern": "^(.*)$" - }, - "title": { - "$id": "#/properties/title", - "type": "string", - "title": "Plugin title", - "description": "Plugin title to display in WIPP forms", - "default": "", - "examples": [ - "WIPP Plugin example" - ], - "minLength": 1, - "pattern": "^(.*)$" - }, - "description": { - "$id": "#/properties/description", - "type": "string", - "title": "Description", - "title": "Short description of the plugin", - "default": "", - "examples": [ - "Custom image segmentation plugin" - ], - "minLength": 1, - "pattern": "^(.*)$" - }, - "author": { - "$id": "#/properties/author", - "type": ["string", "null"], - "title": "Author(s)", - "default": "", - "examples": [ - "FirstName LastName" - ], - "pattern": "^(.*)$" - }, - "institution": { - "$id": "#/properties/institution", - "type": ["string", "null"], - "title": "Institution", - "default": "", - "examples": [ - "National Institute of Standards and Technology" - ], - "pattern": "^(.*)$" - }, - "repository": { - "$id": "#/properties/repository", - "type": ["string", "null"], - "title": "Source code repository", - "default": "", - "examples": [ - "https://github.com/usnistgov/WIPP" - ], - "format": "uri" - }, - "website": { - "$id": "#/properties/website", - "type": ["string", "null"], - "title": "Website", - "default": "", - "examples": [ - "http://usnistgov.github.io/WIPP" - ], - "format": "uri" - }, - "citation": { - "$id": "#/properties/citation", - "type": ["string", "null"], - "title": "Citation", - "default": "", - "examples": [ - "Peter Bajcsy, Joe Chalfoun, and Mylene Simon (2018). Web Microanalysis of Big Image Data. Springer-Verlag International" - ], - "pattern": "^(.*)$" - }, - "containerId": { - "$id": "#/properties/containerId", - "type": "string", - "title": "ContainerId", - "description": "Docker image ID", - "default": "", - "examples": [ - "docker.io/wipp/plugin-example:1.0.0" - ], - "pattern": "^(.*)$" - }, - "baseCommand": { - "$id": "#/properties/baseCommand", - "type": "array", - "title": "Base command", - "description": "Base command to use while running container image", - "default": null, - "items": { - "type": "string" - }, - "examples": [ - ["python3", "/opt/executable/main.py"] - ] - }, - "inputs": { - "$id": "#/properties/inputs", - "type": "array", - "title": "List of Inputs", - "description": "Defines inputs to the plugin", - "default": null, - "uniqueItems": true, - "items": { - "$id": "#/properties/inputs/items", - "type": "object", - "title": "Input", - "description": "Plugin input", - "default": null, - "required": [ - "name", - "type", - "description" - ], - "properties": { - "name": { - "$id": "#/properties/inputs/items/properties/name", - "type": "string", - "title": "Input name", - "description": "Input name as expected by the plugin CLI", - "default": "", - "examples": [ - "inputImages", - "fileNamePattern", - "thresholdValue" - ], - "pattern": "^[a-zA-Z0-9][-a-zA-Z0-9]*$" - }, - "type": { - "$id": "#/properties/inputs/items/properties/type", - "type": "string", - "enum": [ - "collection", - "stitchingVector", - "tensorflowModel", - "csvCollection", - "pyramid", - "pyramidAnnotation", - "notebook", - "genericData", - "string", - "number", - "integer", - "enum", - "array", - "boolean" - ], - "title": "Input Type", - "examples": [ - "collection", - "string", - "number" - ] - }, - "description": { - "$id": "#/properties/inputs/items/properties/description", - "type": "string", - "title": "Input description", - "examples": [ - "Input Images" - ], - "pattern": "^(.*)$" - }, - "required": { - "$id": "#/properties/inputs/items/properties/required", - "type": "boolean", - "title": "Required input", - "description": "Whether an input is required or not", - "default": true, - "examples": [ - true - ] - } - }, - "allOf": [ - { - "if": { - "properties": { "type": { "const": "enum" } } - }, - "then": { - "properties": - { - "options": - { - "$id": "#/properties/inputs/items/properties/options", - "type": "object", - "title": "Input options", - "properties": - { - "values": - { - "type": "array", - "description": "List of possible values", - "items": - { - "type": "string" - }, - "uniqueItems": true - } - } - } - } - } - }, - { - "if": { - "properties": { "type": { "const": "array" } } - }, - "then": { - "properties": - { - "options": - { - "$id": "#/properties/inputs/items/properties/options", - "type": "object", - "title": "Input options", - "properties": - { - "items": { - "$id": "#/properties/inputs/items/properties/options/properties/items", - "type": "object", - "title": "List of array items", - "description": "Possible values for the input array", - "default": {}, - "required": [ - "type", - "title", - "oneOf", - "default", - "widget", - "minItems", - "uniqueItems" - ], - "properties": { - "type": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/type", - "type": "string", - "title": "Items type", - "description": "Type of the items to be selected", - "enum": ["string"], - "examples": [ - "string" - ] - }, - "title": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/title", - "type": "string", - "title": "Selection title", - "description": "Title of the item selection section in the form", - "default": "", - "examples": [ - "Select feature" - ] - }, - "oneOf": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/oneOf", - "type": "array", - "title": "Possible items", - "description": "List of possible items", - "default": [], - "items": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/oneOf/items", - "type": "object", - "title": "Items definition", - "description": "Description of the possible items", - "default": {}, - "required": [ - "description", - "enum" - ], - "properties": { - "description": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/oneOf/items/properties/description", - "type": "string", - "title": "Description", - "description": "Description of the value that will appear in the form", - "default": "", - "examples": [ - "Area" - ] - }, - "enum": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/oneOf/items/properties/enum", - "type": "array", - "title": "Value", - "description": "Values of the selected item", - "default": [], - "items": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/oneOf/items/properties/enum/items", - "type": "string", - "title": "List of values", - "description": "List of values associated with the selected item (usually one value)", - "default": "", - "examples": [ - "Feature2DJava_Area" - ] - } - } - }, - "examples": [ - { - "description": "Area", - "enum": [ - "Feature2DJava_Area" - ] - }, - { - "enum": [ - "Feature2DJava_Mean" - ], - "description": "Mean" - } - ] - } - }, - "default": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/default", - "type": "string", - "title": "Default value", - "description": "Value selected by default (must be one of the possible values)", - "default": "", - "examples": [ - "Feature2DJava_Area" - ] - }, - "widget": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/widget", - "type": "string", - "title": "Item selection widget", - "description": "How items can be selected (select -> dropdown list with add/remove buttons, checkbox -> multi-selection from list)", - "enum": ["select", "checkbox"], - "examples": [ - "select" - ] - }, - "minItems": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/minItems", - "type": "integer", - "title": "Minumum number of items", - "description": "Minumum number of items", - "default": 0, - "examples": [ - 1 - ] - }, - "uniqueItems": { - "$id": "#/properties/inputs/items/properties/options/properties/items/properties/uniqueItems", - "type": ["string", "boolean"], - "title": "Uniqueness of the items", - "description": "Whether items in the array have to be unique", - "examples": [ - "true", true - ] - } - }, - "examples": [ - { - "type": "string", - "widget": "select", - "uniqueItems": "true", - "default": "Feature2DJava_Area", - "minItems": 1, - "title": "Select feature", - "oneOf": [ - { - "description": "Area", - "enum": [ - "Feature2DJava_Area" - ] - }, - { - "description": "Mean", - "enum": [ - "Feature2DJava_Mean" - ] - } - ] - } - ] - } - } - } - } - } - } - ] - } - }, - "outputs": { - "$id": "#/properties/outputs", - "type": "array", - "title": "List of Outputs", - "description": "Defines the outputs of the plugin", - "default": null, - "items": { - "$id": "#/properties/outputs/items", - "type": "object", - "title": "Plugin output", - "default": null, - "required": [ - "name", - "type", - "description" - ], - "properties": { - "name": { - "$id": "#/properties/outputs/items/properties/name", - "type": "string", - "title": "Output name", - "default": "", - "examples": [ - "outputCollection" - ], - "pattern": "^[a-zA-Z0-9][-a-zA-Z0-9]*$" - }, - "type": { - "$id": "#/properties/outputs/items/properties/type", - "type": "string", - "enum": [ - "collection", - "stitchingVector", - "tensorflowModel", - "tensorboardLogs", - "csvCollection", - "pyramid", - "pyramidAnnotation", - "genericData" - ], - "title": "Output type", - "examples": [ - "stitchingVector", - "collection" - ] - }, - "description": { - "$id": "#/properties/outputs/items/properties/description", - "type": "string", - "title": "Output description", - "examples": [ - "Output collection" - ], - "pattern": "^(.*)$" - } - } - } - }, - "ui": { - "$id": "#/properties/ui", - "type": "array", - "title": "Plugin form UI definition", - "items": - { - "type": "object", - "title": "List of UI definitions", - "required": [ - "key" - ], - "properties": { - "key": { - "$id": "#/properties/ui/items/properties/key", - "type": "string", - "title": "UI key", - "description": "Key of the input which this UI definition applies to, the expected format is 'inputs.inputName'. Special keyword 'fieldsets' can be used to define arrangement of inputs by sections.", - "examples": [ - "inputs.inputImages", "inputs.fileNamPattern", "fieldsets" - ], - "oneOf": [ - {"pattern": "^inputs\\.[a-zA-Z0-9][-a-zA-Z0-9]*$"}, - {"const": "fieldsets"} - ] - } - }, - "allOf": [ - { - "if": { - "properties": { "key": { "pattern": "^inputs\\.[a-zA-Z0-9][-a-zA-Z0-9]*$" } } - }, - "then": { - "properties": - { - "title": { - "$id": "#/properties/ui/items/properties/title", - "type": "string", - "title": "Input label", - "default": "", - "examples": [ - "Input images: " - ], - "pattern": "^(.*)$" - }, - "description": { - "$id": "#/properties/ui/items/properties/description", - "type": "string", - "title": "Input placeholder", - "default": "", - "examples": [ - "Pick a collection..." - ], - "pattern": "^(.*)$" - }, - "condition": { - "$id": "#/properties/ui/items/properties/condition", - "type": "string", - "title": "Input visibility condition", - "description": "Definition of when this field is visible or not, depending on the value of another input, the expected format for the condition is 'model.inputs.inputName==value'", - "default": "", - "examples": [ - "model.inputs.thresholdtype=='Manual'" - ], - "pattern": "^(.*)$" - }, - "default": { - "$id": "#/properties/ui/items/properties/default", - "type": ["string", "number", "integer", "boolean"], - "title": "Input default value", - "default": "", - "examples": [ - 5, false, ".ome.tif" - ] - }, - "hidden": { - "$id": "#/properties/ui/items/properties/hidden", - "type": "boolean", - "title": "Hidden input", - "description": "Hidden input will not be displayed on the form, but can be used in conjunction with the 'default' or 'bind' properties to define default or automatically set parameters", - "default": false, - "examples": [ - true, false - ] - }, - "bind": { - "$id": "#/properties/ui/items/properties/bind", - "type": "string", - "title": "Bind input value to another input", - "examples": [ - "gridWidth" - ] - } - }, - "required": [ - "title" - ] - } - }, - { - "if": { - "properties": { "key": { "const": "fieldsets" } } - }, - "then": { - "properties": - { - "fieldsets": - { - "description": "A list of definitions representing sections of input fields.", - "type": "array", - "items": { - "description": "A section of input fields.", - "type": "object", - "properties": { - "title": { - "type": "string", - "description": "The label of the section.", - "examples": [ - "Input images selection" - ] - }, - "fields": { - "description": "A list of input names representing input fields that belong to this section.", - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true, - "minItems": 1, - "examples": [ - "inputImages, fileNamePattern" - ] - } - }, - "uniqueItems": true, - "default": [], - "minItems": 1, - "required": [ - "title", "fields" - ] - } - } - }, - "required": [ - "fieldsets" - ] - } - } - ] - } - }, - "resourceRequirements": { - "type": "object", - "default": {}, - "title": "Plugin Resource Requirements", - "properties": { - "ramMin": { - "type": "number", - "title": "Minimum RAM in mebibytes (Mi)", - "examples": [ - 2048 - ] - }, - "coresMin": { - "type": "number", - "title": "Minimum number of CPU cores", - "examples": [ - 1 - ] - }, - "cpuAVX": { - "type": "boolean", - "default": false, - "title": "Advanced Vector Extensions (AVX) CPU capability required", - "examples": [ - true - ] - }, - "cpuAVX2": { - "type": "boolean", - "default": false, - "title": "Advanced Vector Extensions 2 (AVX2) CPU capability required", - "examples": [ - false - ] - }, - "gpu": { - "type": "boolean", - "default": false, - "title": "GPU/accelerator required", - "examples": [ - true - ] - }, - "cudaRequirements": { - "type": "object", - "default": {}, - "title": "GPU Cuda-related requirements", - "properties": { - "deviceMemoryMin": { - "type": "number", - "default": 0, - "title": "Minimum device memory", - "examples": [ - 100 - ] - }, - "cudaComputeCapability": { - "type": ["string", "array"], - "default": null, - "title": "The cudaComputeCapability Schema", - "description": "Specify either a single minimum value, or an array of valid values", - "examples": [ - "8.0", - ["3.5", "5.0", "6.0", "7.0", "7.5", "8.0"] - ] - } - }, - "examples": [{ - "deviceMemoryMin": 100, - "cudaComputeCapability": "8.0" - }] - } - }, - "examples": [{ - "ramMin": 2048, - "coresMin": 1, - "cpuAVX": true, - "cpuAVX2": false, - "gpu": true, - "cudaRequirements": { - "deviceMemoryMin": 100, - "cudaComputeCapability": "8.0" - } - }] - } - } - } diff --git a/src/polus/plugins/_plugins/registry.py b/src/polus/plugins/_plugins/registry.py deleted file mode 100644 index e09f9a1c3..000000000 --- a/src/polus/plugins/_plugins/registry.py +++ /dev/null @@ -1,280 +0,0 @@ -"""Methods to interact with REST API of WIPP Plugin Registry.""" -import json -import logging -import typing -from urllib.error import HTTPError -from urllib.parse import urljoin - -import requests -import xmltodict -from tqdm import tqdm - -from polus.plugins._plugins.classes import ComputePlugin, Plugin, refresh, submit_plugin -from polus.plugins._plugins.registry_utils import _generate_query, _to_xml - -logger = logging.getLogger("polus.plugins") - - -class FailedToPublish(Exception): - """Raised when there is an error publishing a resource.""" - - -class MissingUserInfo(Exception): - """Raised when necessary user info is not provided for authentication.""" - - -class WippPluginRegistry: - """Class that contains methods to interact with the REST API of WIPP Registry.""" - - def __init__( - self, - username: typing.Optional[str] = None, - password: typing.Optional[str] = None, - registry_url: str = "https://wipp-registry.ci.ncats.io", - verify: bool = True, # verify SSL? - ): - """Initialize WippPluginRegistry from username, password, registry url.""" - self.registry_url = registry_url - self.username = username - self.password = password - self.verify = verify - - @classmethod - def _parse_xml(cls, xml: str): - """Return dictionary of Plugin Manifest. If error, return None.""" - d = xmltodict.parse(xml)["Resource"]["role"]["PluginManifest"][ - "PluginManifestContent" - ]["#text"] - try: - return json.loads(d) - except BaseException: - e = eval(d) - if isinstance(e, dict): - return e - else: - return None - - def update_plugins(self): - """Update plugins from WIPP Registry.""" - url = self.registry_url + "/rest/data/query/" - headers = {"Content-type": "application/json"} - data = '{"query": {"$or":[{"Resource.role.type":"Plugin"},{"Resource.role.type.#text":"Plugin"}]}}' - if self.username and self.password: - r = requests.post( - url, - headers=headers, - data=data, - auth=(self.username, self.password), - verify=self.verify, - ) # authenticated request - else: - r = requests.post(url, headers=headers, data=data, verify=self.verify) - valid, invalid = 0, {} - - for r in tqdm(r.json()["results"], desc="Updating Plugins from WIPP"): - try: - manifest = WippPluginRegistry._parse_xml(r["xml_content"]) - submit_plugin(manifest) - valid += 1 - except BaseException as err: - invalid.update({r["title"]: err.args[0]}) - - finally: - if len(invalid) > 0: - self.invalid = invalid - logger.debug( - "Submitted %s plugins successfully. See WippPluginRegistry.invalid to check errors in unsubmitted plugins" - % (valid) - ) - logger.debug("Submitted %s plugins successfully." % (valid)) - refresh() - - def query( - self, - title: typing.Optional[str] = None, - version: typing.Optional[str] = None, - title_contains: typing.Optional[str] = None, - contains: typing.Optional[str] = None, - query_all: bool = False, - advanced: bool = False, - query: typing.Optional[str] = None, - ): - """Query Plugins in WIPP Registry. - - This function executes queries for Plugins in the WIPP Registry. - - Args: - title: - title of the plugin to query. - Example: "OME Tiled Tiff Converter" - version: - version of the plugins to query. - Must follow semantic versioning. Example: "1.1.0" - title_contains: - keyword that must be part of the title of plugins to query. - Example: "Converter" will return all plugins with the word "Converter" in their title - contains: - keyword that must be part of the description of plugins to query. - Example: "bioformats" will return all plugins with the word "bioformats" in their description - query_all: if True it will override any other parameter and will return all plugins - advanced: - if True it will override any other parameter. - `query` must be included - query: query to execute. This query must be in MongoDB format - - - Returns: - An array of the manifests of the Plugins returned by the query. - """ - url = self.registry_url + "/rest/data/query/" - headers = {"Content-type": "application/json"} - query = _generate_query( - title, version, title_contains, contains, query_all, advanced, query - ) - - data = '{"query": %s}' % str(query).replace("'", '"') - - if self.username and self.password: - r = requests.post( - url, - headers=headers, - data=data, - auth=(self.username, self.password), - verify=self.verify, - ) # authenticated request - else: - r = requests.post(url, headers=headers, data=data, verify=self.verify) - logger.debug(f"r is {r.raise_for_status}") - return [ - WippPluginRegistry._parse_xml(x["xml_content"]) for x in r.json()["results"] - ] - - def get_current_schema(self): - """Return current schema in WIPP.""" - r = requests.get( - urljoin( - self.registry_url, - "rest/template-version-manager/global/?title=res-md.xsd", - ), - verify=self.verify, - ) - if r.ok: - return r.json()[0]["current"] - else: - r.raise_for_status() - - def upload( - self, - plugin: typing.Union[Plugin, ComputePlugin], - author: typing.Optional[str] = None, - email: typing.Optional[str] = None, - publish: bool = True, - ): - """Upload Plugin to WIPP Registry. - - This function uploads a Plugin object to the WIPP Registry. - Author name and email to be passed to the Plugin object - information on the WIPP Registry are taken from the value - of the field `author` in the `Plugin` manifest. That is, - the first email and the first name (first and last) will - be passed. The value of these two fields can be overridden - by specifying them in the arguments. - - Args: - plugin: - Plugin to be uploaded - author: - Optional `str` to override author name - email: - Optional `str` to override email - publish: - If `False`, Plugin will not be published to the public - workspace. It will be visible only to the user uploading - it. Default is `True` - - Returns: - A message indicating a successful upload. - """ - manifest = plugin.manifest - - xml_content = _to_xml(manifest, author, email) - - schema_id = self.get_current_schema() - - data = { - "title": manifest["name"], - "template": schema_id, - "xml_content": xml_content, - } - - url = self.registry_url + "/rest/data/" - headers = {"Content-type": "application/json"} - if self.username and self.password: - r = requests.post( - url, - headers=headers, - data=json.dumps(data), - auth=(self.username, self.password), - verify=self.verify, - ) # authenticated request - else: - raise MissingUserInfo("The registry connection must be authenticated.") - - response_code = r.status_code - - if response_code != 201: - print( - "Error uploading file (%s), code %s" - % (data["title"], str(response_code)) - ) - r.raise_for_status() - if publish: - _id = r.json()["id"] - _purl = url + _id + "/publish/" - r2 = requests.patch( - _purl, - headers=headers, - auth=(self.username, self.password), - verify=self.verify, - ) - try: - r2.raise_for_status() - except HTTPError as err: - raise FailedToPublish( - "Failed to publish {} with id {}".format(data["title"], _id) - ) from err - - return "Successfully uploaded %s" % data["title"] - - def get_resource_by_pid(self, pid): - """Return current resource.""" - response = requests.get(pid, verify=self.verify) - return response.json() - - def patch_resource( - self, - pid, - version, - ): - """Patch resource in registry.""" - if self.username is None or self.password is None: - raise MissingUserInfo("The registry connection must be authenticated.") - - # Get current version of the resource - data = self.get_resource_by_pid(pid) - - data.update({"version": version}) - response = requests.patch( - urljoin(self.registry_url, "rest/data/" + data["id"]), - data, - auth=(self.username, self.password), - verify=self.verify, - ) - response_code = response.status_code - - if response_code != 200: - print( - "Error publishing data (%s), code %s" - % (data["title"], str(response_code)) - ) - response.raise_for_status() diff --git a/src/polus/plugins/_plugins/registry_utils.py b/src/polus/plugins/_plugins/registry_utils.py deleted file mode 100644 index 8c2bd5163..000000000 --- a/src/polus/plugins/_plugins/registry_utils.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Utilities for WIPP Registry Module.""" -import re -import typing - - -def _generate_query( - title, version, title_contains, contains, query_all, advanced, query -): - if advanced: - if not query: - raise ValueError("query cannot be empty if advanced is True") - else: - return query - if query_all: - q = { - "$or": [ - {"Resource.role.type": "Plugin"}, - {"Resource.role.type.#text": "Plugin"}, - ] - } # replace query - return q - - # Check for possible errors: - if title and title_contains: - raise ValueError("Cannot define title and title_contains together") - q = {} # query to return - q["$and"] = [] - q["$and"].append( - { - "$or": [ - {"Resource.role.type": "Plugin"}, - {"Resource.role.type.#text": "Plugin"}, - ] - } - ) - if title: - q["$and"].append( - { - "$or": [ - {"Resource.identity.title.#text": title}, - {"Resource.identity.title": title}, - ] - } - ) - if version: - q["$and"].append( - { - "$or": [ - {"Resource.identity.version.#text": version}, - {"Resource.identity.version": version}, - ] - } - ) - if contains: - q["$and"].append( - { - "$or": [ - { - "Resource.content.description.#text": { - "$regex": f".*{contains}.*", - "$options": "i", - } - }, - { - "Resource.content.description": { - "$regex": f".*{contains}.*", - "$options": "i", - } - }, - ] - } - ) - if title_contains: - q["$and"].append( - { - "$or": [ - { - "Resource.identity.title.#text": { - "$regex": f".*{title_contains}.*", - "$options": "i", - } - }, - { - "Resource.identity.title": { - "$regex": f".*{title_contains}.*", - "$options": "i", - } - }, - ] - } - ) - return q - - -def _get_email(author: str): - regex = re.compile(r"[A-Za-z][A-Za-z0-9.]*@[A-Za-z0-9.]*") - return regex.search(author).group() - - -def _get_author(author: str): - return " ".join(author.split()[0:2]) - - -def _to_xml( - manifest: dict, - author: typing.Optional[str] = None, - email: typing.Optional[str] = None, -): - if email is None: - email = _get_email(manifest["author"]) - if author is None: - author = _get_author(manifest["author"]) - - xml = ( - '' - f'{manifest["name"]}' - f'{str(manifest["version"])}' - '' - f'{manifest["institution"]}' - '' - f'{author}' - f'{email}' - '' - f'{manifest["description"]}' - '' - 'Plugin' - f'{manifest["containerId"]}' - '' - f'{str(manifest)}' - ) - - return xml diff --git a/src/polus/plugins/_plugins/update/__init__.py b/src/polus/plugins/_plugins/update/__init__.py deleted file mode 100644 index 936f7b803..000000000 --- a/src/polus/plugins/_plugins/update/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Initialize update module.""" - -from polus.plugins._plugins.update._update import update_nist_plugins -from polus.plugins._plugins.update._update import update_polus_plugins - -__all__ = ["update_polus_plugins", "update_nist_plugins"] diff --git a/src/polus/plugins/_plugins/update/_update.py b/src/polus/plugins/_plugins/update/_update.py deleted file mode 100644 index cc184d493..000000000 --- a/src/polus/plugins/_plugins/update/_update.py +++ /dev/null @@ -1,116 +0,0 @@ -# pylint: disable=W1203, W1201 -import json -import logging -import re -import typing - -from polus.plugins._plugins._compat import PYDANTIC_V2 -from polus.plugins._plugins.classes import refresh -from polus.plugins._plugins.classes import submit_plugin -from polus.plugins._plugins.gh import _init_github -from polus.plugins._plugins.io import Version -from polus.plugins._plugins.manifests import _error_log -from polus.plugins._plugins.manifests import _scrape_manifests -from pydantic import ValidationError -from tqdm import tqdm - -logger = logging.getLogger("polus.plugins") - - -def update_polus_plugins( - gh_auth: typing.Optional[str] = None, - min_depth: int = 2, - max_depth: int = 3, -) -> None: - """Scrape PolusAI GitHub repo and create local versions of Plugins.""" - logger.info("Updating polus plugins.") - # Get all manifests - valid, invalid = _scrape_manifests( - "polusai/polus-plugins", - _init_github(gh_auth), - min_depth, - max_depth, - True, - ) - manifests = valid.copy() - manifests.extend(invalid) - logger.info(f"Submitting {len(manifests)} plugins.") - - for manifest in manifests: - try: - plugin = submit_plugin(manifest) - - # Parsing checks specific to polus-plugins - error_list = [] - - # Check that plugin version matches container version tag - container_name, version = tuple(plugin.containerId.split(":")) - version = Version(version) if PYDANTIC_V2 else Version(version=version) - organization, container_name = tuple(container_name.split("/")) - if plugin.version != version: - msg = ( - f"containerId version ({version}) does not " - f"match plugin version ({plugin.version})" - ) - logger.error(msg) - error_list.append(ValueError(msg)) - - # Check to see that the plugin is registered to Labshare - if organization not in ["polusai", "labshare"]: - msg = ( - "all polus plugin containers must be" - " under the Labshare organization." - ) - logger.error(msg) - error_list.append(ValueError(msg)) - - # Checks for container name, they are somewhat related to our - # Jenkins build - if not container_name.startswith("polus"): - msg = "containerId name must begin with polus-" - logger.error(msg) - error_list.append(ValueError(msg)) - - if not container_name.endswith("plugin"): - msg = "containerId name must end with -plugin" - logger.error(msg) - error_list.append(ValueError(msg)) - - if len(error_list) > 0: - raise ValidationError(error_list, plugin.__class__) - - except ValidationError as val_err: - try: - _error_log(val_err, manifest, "update_polus_plugins") - except BaseException as e: # pylint: disable=W0718 - logger.exception(f"In {plugin.name}: {e}") - except BaseException as e: # pylint: disable=W0718 - logger.exception(f"In {plugin.name}: {e}") - refresh() - - -def update_nist_plugins(gh_auth: typing.Optional[str] = None) -> None: - """Scrape NIST GitHub repo and create local versions of Plugins.""" - # Parse README links - gh = _init_github(gh_auth) - repo = gh.get_repo("usnistgov/WIPP") - contents = repo.get_contents("plugins") - readme = [r for r in contents if r.name == "README.md"][0] - pattern = re.compile( - r"\[manifest\]\((https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*))\)", - ) - matches = pattern.findall(str(readme.decoded_content)) - logger.info("Updating NIST plugins.") - for match in tqdm(matches, desc="NIST Manifests"): - url_parts = match[0].split("/")[3:] - plugin_repo = gh.get_repo("/".join(url_parts[:2])) - manifest = json.loads( - plugin_repo.get_contents("/".join(url_parts[4:])).decoded_content, - ) - - try: - submit_plugin(manifest) - - except ValidationError as val_err: - _error_log(val_err, manifest, "update_nist_plugins") - refresh() diff --git a/src/polus/plugins/_plugins/utils.py b/src/polus/plugins/_plugins/utils.py deleted file mode 100644 index 6d9b19ddb..000000000 --- a/src/polus/plugins/_plugins/utils.py +++ /dev/null @@ -1,17 +0,0 @@ -"""General utilities for polus-plugins.""" -from polus.plugins._plugins.io import Version - - -def name_cleaner(name: str) -> str: - """Generate Plugin Class Name from Plugin name in manifest.""" - replace_chars = "()<>-_" - for char in replace_chars: - name = name.replace(char, " ") - return name.title().replace(" ", "").replace("/", "_") - - -def cast_version(value): - """Return Version object from version str or dict.""" - if isinstance(value, dict): # if init from a Version object - value = value["version"] - return Version(version=value) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 4ede8e6df..000000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# noqa diff --git a/tests/resources/b1.json b/tests/resources/b1.json deleted file mode 100644 index a385c3baf..000000000 --- a/tests/resources/b1.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "version": "1.2.7", - "title": "Flatfield correction using BaSiC algorithm.", - "description": "Generates images used for flatfield correction using the BaSiC algorithm.", - "author": "Nick Schaub (nick.schaub@nih.gov)", - "institution": "National Center for the Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/polusai/polus-plugins", - "website": "https://ncats.nih.gov/preclinical/core/informatics", - "citation": "Peng et al. \"A BaSiC tool for background and shading correction of optical microscopy images\" Nature Communications (2017)", - "containerId": "polusai/basic-flatfield-correction-plugin:1.2.7", - "inputs": [ - { - "name": "inpDir", - "type": "collection", - "description": "Input image collection.", - "required": true - }, - { - "name": "filePattern", - "type": "string", - "description": "Filename pattern used to separate images by channel, timepoint, and replicate.", - "required": true - }, - { - "name": "darkfield", - "type": "boolean", - "description": "Calculate darkfield image.", - "required": true - }, - { - "name": "photobleach", - "type": "boolean", - "description": "Calculate photobleaching offsets.", - "required": true - }, - { - "name": "groupBy", - "type": "string", - "description": "Group images together for flatfield by variable.", - "required": false - } - ], - "outputs": [ - { - "name": "outDir", - "type": "collection", - "description": "Output data for the plugin" - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input image collection: ", - "description": "Image collection..." - }, - { - "key": "inputs.filePattern", - "title": "Filename pattern: ", - "description": "Use a filename pattern to calculate flatfield information by subsets" - }, - { - "key": "inputs.groupBy", - "title": "Grouping Variables: ", - "description": "Group data together with varying variable values." - }, - { - "key": "inputs.darkfield", - "title": "Calculate darkfield: ", - "description": "If selected, will generate a darkfield image" - }, - { - "key": "inputs.photobleach", - "title": "Calclate photobleaching offset: ", - "description": "If selected, will generate an offset scalar for each image" - } - ] -} diff --git a/tests/resources/b2.json b/tests/resources/b2.json deleted file mode 100644 index 3e28f5b1c..000000000 --- a/tests/resources/b2.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "name": "BaSiC Flatfield Correction Plugin", - "version": "1.2.7", - "description": "Generates images used for flatfield correction using the BaSiC algorithm.", - "author": "Nick Schaub (nick.schaub@nih.gov)", - "institution": "National Center for the Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/polusai/polus-plugins", - "citation": "Peng et al. \"A BaSiC tool for background and shading correction of optical microscopy images\" Nature Communications (2017)", - "containerId": "polusai/basic-flatfield-correction-plugin:1.2.7", - "inputs": [ - { - "name": "inpDir", - "type": "collection", - "description": "Input image collection.", - "required": true - }, - { - "name": "filePattern", - "type": "string", - "description": "Filename pattern used to separate images by channel, timepoint, and replicate.", - "required": true - }, - { - "name": "darkfield", - "type": "boolean", - "description": "Calculate darkfield image.", - "required": true - }, - { - "name": "photobleach", - "type": "boolean", - "description": "Calculate photobleaching offsets.", - "required": true - }, - { - "name": "groupBy", - "type": "string", - "description": "Group images together for flatfield by variable.", - "required": false - } - ], - "outputs": [ - { - "name": "outDir", - "type": "collection", - "description": "Output data for the plugin" - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input image collection: ", - "description": "Image collection..." - }, - { - "key": "inputs.filePattern", - "title": "Filename pattern: ", - "description": "Use a filename pattern to calculate flatfield information by subsets" - }, - { - "key": "inputs.groupBy", - "title": "Grouping Variables: ", - "description": "Group data together with varying variable values." - }, - { - "key": "inputs.darkfield", - "title": "Calculate darkfield: ", - "description": "If selected, will generate a darkfield image" - }, - { - "key": "inputs.photobleach", - "title": "Calclate photobleaching offset: ", - "description": "If selected, will generate an offset scalar for each image" - } - ] -} diff --git a/tests/resources/b3.json b/tests/resources/b3.json deleted file mode 100644 index f161974fa..000000000 --- a/tests/resources/b3.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "name": "BaSiC Flatfield Correction Plugin", - "version": "1.2.7", - "title": "Flatfield correction using BaSiC algorithm.", - "author": "Nick Schaub (nick.schaub@nih.gov)", - "institution": "National Center for the Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/polusai/polus-plugins", - "citation": "Peng et al. \"A BaSiC tool for background and shading correction of optical microscopy images\" Nature Communications (2017)", - "containerId": "polusai/basic-flatfield-correction-plugin:1.2.7", - "inputs": [ - { - "name": "inpDir", - "type": "collection", - "description": "Input image collection.", - "required": true - }, - { - "name": "filePattern", - "type": "string", - "description": "Filename pattern used to separate images by channel, timepoint, and replicate.", - "required": true - }, - { - "name": "darkfield", - "type": "boolean", - "description": "Calculate darkfield image.", - "required": true - }, - { - "name": "photobleach", - "type": "boolean", - "description": "Calculate photobleaching offsets.", - "required": true - }, - { - "name": "groupBy", - "type": "string", - "description": "Group images together for flatfield by variable.", - "required": false - } - ], - "outputs": [ - { - "name": "outDir", - "type": "collection", - "description": "Output data for the plugin" - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input image collection: ", - "description": "Image collection..." - }, - { - "key": "inputs.filePattern", - "title": "Filename pattern: ", - "description": "Use a filename pattern to calculate flatfield information by subsets" - }, - { - "key": "inputs.groupBy", - "title": "Grouping Variables: ", - "description": "Group data together with varying variable values." - }, - { - "key": "inputs.darkfield", - "title": "Calculate darkfield: ", - "description": "If selected, will generate a darkfield image" - }, - { - "key": "inputs.photobleach", - "title": "Calclate photobleaching offset: ", - "description": "If selected, will generate an offset scalar for each image" - } - ] -} diff --git a/tests/resources/g1.json b/tests/resources/g1.json deleted file mode 100644 index ca32f1905..000000000 --- a/tests/resources/g1.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "name": "BaSiC Flatfield Correction Plugin", - "version": "1.2.7", - "title": "Flatfield correction using BaSiC algorithm.", - "description": "Generates images used for flatfield correction using the BaSiC algorithm.", - "author": "Nick Schaub (nick.schaub@nih.gov)", - "institution": "National Center for the Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/polusai/polus-plugins", - "website": "https://ncats.nih.gov/preclinical/core/informatics", - "citation": "Peng et al. \"A BaSiC tool for background and shading correction of optical microscopy images\" Nature Communications (2017)", - "containerId": "polusai/basic-flatfield-correction-plugin:1.2.7", - "inputs": [ - { - "name": "inpDir", - "type": "collection", - "description": "Input image collection.", - "required": true - }, - { - "name": "filePattern", - "type": "string", - "description": "Filename pattern used to separate images by channel, timepoint, and replicate.", - "required": true - }, - { - "name": "darkfield", - "type": "boolean", - "description": "Calculate darkfield image.", - "required": true - }, - { - "name": "photobleach", - "type": "boolean", - "description": "Calculate photobleaching offsets.", - "required": true - }, - { - "name": "groupBy", - "type": "string", - "description": "Group images together for flatfield by variable.", - "required": false - } - ], - "outputs": [ - { - "name": "outDir", - "type": "collection", - "description": "Output data for the plugin" - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input image collection: ", - "description": "Image collection..." - }, - { - "key": "inputs.filePattern", - "title": "Filename pattern: ", - "description": "Use a filename pattern to calculate flatfield information by subsets" - }, - { - "key": "inputs.groupBy", - "title": "Grouping Variables: ", - "description": "Group data together with varying variable values." - }, - { - "key": "inputs.darkfield", - "title": "Calculate darkfield: ", - "description": "If selected, will generate a darkfield image" - }, - { - "key": "inputs.photobleach", - "title": "Calclate photobleaching offset: ", - "description": "If selected, will generate an offset scalar for each image" - } - ] -} diff --git a/tests/resources/g2.json b/tests/resources/g2.json deleted file mode 100644 index 24d32be1a..000000000 --- a/tests/resources/g2.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "version": "1.2.7", - "institution": "National Center for the Advancing Translational Sciences, National Institutes of Health", - "description": "Generates images used for flatfield correction using the BaSiC algorithm.", - "name": "BaSiC Flatfield Correction Plugin", - "author": "Nick Schaub (nick.schaub@nih.gov)", - "repository": "https://github.com/polusai/polus-plugins", - "title": "Flatfield correction using BaSiC algorithm.", - "website": "https://ncats.nih.gov/preclinical/core/informatics", - "containerId": "polusai/basic-flatfield-correction-plugin:1.2.7", - "inputs": [ - { - "name": "inpDir", - "type": "collection", - "description": "Input image collection.", - "required": true - }, - { - "name": "filePattern", - "type": "string", - "description": "Filename pattern used to separate images by channel, timepoint, and replicate.", - "required": true - }, - { - "name": "darkfield", - "type": "boolean", - "description": "Calculate darkfield image.", - "required": true - }, - { - "name": "photobleach", - "type": "boolean", - "description": "Calculate photobleaching offsets.", - "required": true - }, - { - "name": "groupBy", - "type": "string", - "description": "Group images together for flatfield by variable.", - "required": false - } - ], - "outputs": [ - { - "name": "outDir", - "type": "collection", - "description": "Output data for the plugin" - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input image collection: ", - "description": "Image collection..." - }, - { - "key": "inputs.filePattern", - "title": "Filename pattern: ", - "description": "Use a filename pattern to calculate flatfield information by subsets" - }, - { - "key": "inputs.groupBy", - "title": "Grouping Variables: ", - "description": "Group data together with varying variable values." - }, - { - "key": "inputs.darkfield", - "title": "Calculate darkfield: ", - "description": "If selected, will generate a darkfield image" - }, - { - "key": "inputs.photobleach", - "title": "Calclate photobleaching offset: ", - "description": "If selected, will generate an offset scalar for each image" - } - ] -} diff --git a/tests/resources/g3.json b/tests/resources/g3.json deleted file mode 100644 index e589644e4..000000000 --- a/tests/resources/g3.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "name": "BaSiC Flatfield Correction Plugin", - "version": "1.2.7", - "title": "Flatfield correction using BaSiC algorithm.", - "description": "Generates images used for flatfield correction using the BaSiC algorithm.", - "institution": "National Center for the Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/polusai/polus-plugins", - "website": "https://ncats.nih.gov/preclinical/core/informatics", - "citation": "Peng et al. \"A BaSiC tool for background and shading correction of optical microscopy images\" Nature Communications (2017)", - "containerId": "polusai/basic-flatfield-correction-plugin:1.2.7", - "inputs": [ - { - "name": "inpDir", - "type": "collection", - "description": "Input image collection.", - "required": true - }, - { - "name": "filePattern", - "type": "string", - "description": "Filename pattern used to separate images by channel, timepoint, and replicate.", - "required": true - }, - { - "name": "darkfield", - "type": "boolean", - "description": "Calculate darkfield image.", - "required": true - }, - { - "name": "photobleach", - "type": "boolean", - "description": "Calculate photobleaching offsets.", - "required": true - }, - { - "name": "groupBy", - "type": "string", - "description": "Group images together for flatfield by variable.", - "required": false - } - ], - "outputs": [ - { - "name": "outDir", - "type": "collection", - "description": "Output data for the plugin" - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input image collection: ", - "description": "Image collection..." - }, - { - "key": "inputs.filePattern", - "title": "Filename pattern: ", - "description": "Use a filename pattern to calculate flatfield information by subsets" - }, - { - "key": "inputs.groupBy", - "title": "Grouping Variables: ", - "description": "Group data together with varying variable values." - }, - { - "key": "inputs.darkfield", - "title": "Calculate darkfield: ", - "description": "If selected, will generate a darkfield image" - }, - { - "key": "inputs.photobleach", - "title": "Calclate photobleaching offset: ", - "description": "If selected, will generate an offset scalar for each image" - } - ] -} diff --git a/tests/resources/omeconverter022.json b/tests/resources/omeconverter022.json deleted file mode 100644 index b696f46e6..000000000 --- a/tests/resources/omeconverter022.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "name": "OME Converter", - "version": "0.2.2", - "title": "OME Converter", - "description": "Convert Bioformats supported format to OME Zarr.", - "author": "Nick Schaub (nick.schaub@nih.gov)", - "institution": "National Center for Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/labshare/polus-plugins", - "website": "https://ncats.nih.gov/preclinical/core/informatics", - "citation": "", - "containerId": "polusai/ome-converter-plugin:0.2.2", - "inputs": [ - { - "name": "inpDir", - "type": "genericData", - "description": "Input generic data collection to be processed by this plugin", - "required": true - }, - { - "name": "filePatter", - "type": "string", - "description": "A filepattern, used to select data to be converted", - "required": true - } - ], - "outputs": [ - { - "name": "outDir", - "type": "genericData", - "description": "Output collection" - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input generic collection", - "description": "Input generic data collection to be processed by this plugin" - }, - { - "key": "inputs.filePattern", - "title": "Filepattern", - "description": "A filepattern, used to select data for conversion" - } - ] -} diff --git a/tests/resources/omeconverter030.json b/tests/resources/omeconverter030.json deleted file mode 100644 index 7613dcf3c..000000000 --- a/tests/resources/omeconverter030.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "name": "OME Converter", - "version": "0.3.0", - "title": "OME Converter", - "description": "Convert Bioformats supported format to OME Zarr or OME TIF", - "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", - "institution": "National Center for Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/PolusAI/polus-plugins", - "website": "https://ncats.nih.gov/preclinical/core/informatics", - "citation": "", - "containerId": "polusai/ome-converter-plugin:0.3.0", - "baseCommand": [ - "python3", - "-m", - "polus.plugins.formats.ome_converter" - ], - "inputs": [ - { - "name": "inpDir", - "type": "genericData", - "description": "Input generic data collection to be processed by this plugin", - "required": true - }, - { - "name": "filePattern", - "type": "string", - "description": "A filepattern, used to select data to be converted", - "required": true - }, - { - "name": "fileExtension", - "type": "enum", - "description": "Type of data conversion", - "default": "default", - "options": { - "values": [ - ".ome.tif", - ".ome.zarr", - "default" - ] - }, - "required": true - } - ], - "outputs": [ - { - "name": "outDir", - "type": "genericData", - "description": "Output collection" - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input generic collection", - "description": "Input generic data collection to be processed by this plugin" - }, - { - "key": "inputs.filePattern", - "title": "Filepattern", - "description": "A filepattern, used to select data for conversion" - }, - { - "key": "inputs.fileExtension", - "title": "fileExtension", - "description": "Type of data conversion", - "default": ".ome.tif" - } - ] -} diff --git a/tests/resources/target1.cwl b/tests/resources/target1.cwl deleted file mode 100644 index 322dfb093..000000000 --- a/tests/resources/target1.cwl +++ /dev/null @@ -1,32 +0,0 @@ -class: CommandLineTool -cwlVersion: v1.2 -inputs: - fileExtension: - inputBinding: - prefix: --fileExtension - type: string - filePattern: - inputBinding: - prefix: --filePattern - type: string - inpDir: - inputBinding: - prefix: --inpDir - type: Directory - outDir: - inputBinding: - prefix: --outDir - type: Directory -outputs: - outDir: - outputBinding: - glob: $(inputs.outDir.basename) - type: Directory -requirements: - DockerRequirement: - dockerPull: polusai/ome-converter-plugin:0.3.0 - InitialWorkDirRequirement: - listing: - - entry: $(inputs.outDir) - writable: true - InlineJavascriptRequirement: {} diff --git a/tests/test_cwl.py b/tests/test_cwl.py deleted file mode 100644 index 58ac9fd08..000000000 --- a/tests/test_cwl.py +++ /dev/null @@ -1,105 +0,0 @@ -# type: ignore -# pylint: disable=W0621, W0613 -"""Tests for CWL utils.""" -from pathlib import Path - -import pydantic -import pytest -import yaml - -import polus.plugins as pp -from polus.plugins._plugins.classes.plugin_base import MissingInputValuesError - -PYDANTIC_VERSION = pydantic.__version__.split(".")[0] -RSRC_PATH = Path(__file__).parent.joinpath("resources") - -OMECONVERTER = RSRC_PATH.joinpath("omeconverter030.json") - - -@pytest.fixture -def submit_plugin(): - """Submit OmeConverter plugin.""" - if "OmeConverter" not in pp.list: - pp.submit_plugin(OMECONVERTER) - else: - if "0.3.0" not in pp.OmeConverter.versions: - pp.submit_plugin(OMECONVERTER) - - -@pytest.fixture -def plug(submit_plugin): - """Get OmeConverter plugin.""" - return pp.get_plugin("OmeConverter", "0.3.0") - - -@pytest.fixture(scope="session") -def cwl_io_path(tmp_path_factory): - """Temp CWL IO path.""" - return tmp_path_factory.mktemp("io") / "omeconverter_io.yml" - - -@pytest.fixture(scope="session") -def cwl_path(tmp_path_factory): - """Temp CWL IO path.""" - return tmp_path_factory.mktemp("cwl") / "omeconverter.cwl" - - -@pytest.fixture -def cwl_io(plug, cwl_io_path): - """Test save_cwl IO.""" - rs_path = RSRC_PATH.absolute() - plug.inpDir = rs_path - plug.filePattern = "img_r{rrr}_c{ccc}.tif" - plug.fileExtension = ".ome.zarr" - plug.outDir = rs_path - plug.save_cwl_io(cwl_io_path) - - -def test_save_read_cwl(plug, cwl_path): - """Test save and read cwl.""" - plug.save_cwl(cwl_path) - with open(cwl_path, encoding="utf-8") as file: - src_cwl = file.read() - with open(RSRC_PATH.joinpath("target1.cwl"), encoding="utf-8") as file: - target_cwl = file.read() - assert src_cwl == target_cwl - - -def test_save_cwl_io_not_inp(plug, cwl_io_path): - """Test save_cwl IO.""" - with pytest.raises(MissingInputValuesError): - plug.save_cwl_io(cwl_io_path) - - -def test_save_cwl_io_not_inp2(plug, cwl_io_path): - """Test save_cwl IO.""" - plug.inpDir = RSRC_PATH.absolute() - plug.filePattern = "img_r{rrr}_c{ccc}.tif" - with pytest.raises(MissingInputValuesError): - plug.save_cwl_io(cwl_io_path) - - -def test_save_cwl_io_not_yml(plug, cwl_io_path): - """Test save_cwl IO.""" - plug.inpDir = RSRC_PATH.absolute() - plug.filePattern = "img_r{rrr}_c{ccc}.tif" - plug.fileExtension = ".ome.zarr" - plug.outDir = RSRC_PATH.absolute() - with pytest.raises(ValueError): - plug.save_cwl_io(cwl_io_path.with_suffix(".txt")) - - -def test_read_cwl_io(cwl_io, cwl_io_path): - """Test read_cwl_io.""" - with open(cwl_io_path, encoding="utf-8") as file: - src_io = yaml.safe_load(file) - assert src_io["inpDir"] == { - "class": "Directory", - "location": str(RSRC_PATH.absolute()), - } - assert src_io["outDir"] == { - "class": "Directory", - "location": str(RSRC_PATH.absolute()), - } - assert src_io["filePattern"] == "img_r{rrr}_c{ccc}.tif" - assert src_io["fileExtension"] == ".ome.zarr" diff --git a/tests/test_io.py b/tests/test_io.py deleted file mode 100644 index 83e289a6e..000000000 --- a/tests/test_io.py +++ /dev/null @@ -1,69 +0,0 @@ -# pylint: disable=C0103 -"""IO Tests.""" -from pathlib import Path - -import pytest -from fsspec.implementations.local import LocalFileSystem - -from polus.plugins._plugins.classes import _load_plugin -from polus.plugins._plugins.classes.plugin_base import IOKeyError -from polus.plugins._plugins.io import Input, IOBase - -RSRC_PATH = Path(__file__).parent.joinpath("resources") - -io1 = { - "type": "collection", - "name": "input1", - "required": True, - "description": "Test IO", -} -io2 = {"type": "boolean", "name": "input2", "required": True, "description": "Test IO"} -iob1 = { - "type": "collection", -} -plugin = _load_plugin(RSRC_PATH.joinpath("g1.json")) - - -def test_iobase(): - """Test IOBase.""" - IOBase(**iob1) - - -@pytest.mark.parametrize("io", [io1, io2], ids=["io1", "io2"]) -def test_input(io): - """Test Input.""" - Input(**io) - - -def test_set_attr_invalid1(): - """Test setting invalid attribute.""" - with pytest.raises(TypeError): - plugin.inputs[0].examples = [2, 5] - - -def test_set_attr_invalid2(): - """Test setting invalid attribute.""" - with pytest.raises(IOKeyError): - plugin.invalid = False - - -def test_set_attr_valid1(): - """Test setting valid attribute.""" - i = [x for x in plugin.inputs if x.name == "darkfield"] - i[0].value = True - - -def test_set_attr_valid2(): - """Test setting valid attribute.""" - plugin.darkfield = True - - -def test_set_fsspec(): - """Test setting fs valid attribute.""" - plugin._fs = LocalFileSystem() # pylint: disable=protected-access - - -def test_set_fsspec2(): - """Test setting fs invalid attribute.""" - with pytest.raises(ValueError): - plugin._fs = "./" # pylint: disable=protected-access diff --git a/tests/test_manifests.py b/tests/test_manifests.py deleted file mode 100644 index d24a5d91f..000000000 --- a/tests/test_manifests.py +++ /dev/null @@ -1,236 +0,0 @@ -# pylint: disable=C0103 -"""Test manifests utils.""" -from collections import OrderedDict -from pathlib import Path - -import pytest - -from polus.plugins._plugins.classes import PLUGINS, list_plugins -from polus.plugins._plugins.manifests import ( - InvalidManifestError, - _load_manifest, - validate_manifest, -) -from polus.plugins._plugins.models import ComputeSchema, WIPPPluginManifest - -RSRC_PATH = Path(__file__).parent.joinpath("resources") - -d_val = { - "name": "BaSiC Flatfield Correction Plugin", - "version": "1.2.7", - "title": "Flatfield correction using BaSiC algorithm.", - "description": "Generates images used for flatfield correction using the BaSiC algorithm.", - "author": "Nick Schaub (nick.schaub@nih.gov)", - "institution": "National Center for the Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/polusai/polus-plugins", - "website": "https://ncats.nih.gov/preclinical/core/informatics", - "citation": 'Peng et al. "A BaSiC tool for background and shading correction of optical microscopy images" Nature Communications (2017)', - "containerId": "polusai/basic-flatfield-correction-plugin:1.2.7", - "inputs": [ - { - "name": "inpDir", - "type": "collection", - "description": "Input image collection.", - "required": True, - }, - { - "name": "filePattern", - "type": "string", - "description": "Filename pattern used to separate images by channel, timepoint, and replicate.", - "required": True, - }, - { - "name": "darkfield", - "type": "boolean", - "description": "Calculate darkfield image.", - "required": True, - }, - { - "name": "photobleach", - "type": "boolean", - "description": "Calculate photobleaching offsets.", - "required": True, - }, - { - "name": "groupBy", - "type": "string", - "description": "Group images together for flatfield by variable.", - "required": False, - }, - ], - "outputs": [ - { - "name": "outDir", - "type": "collection", - "description": "Output data for the plugin", - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input image collection: ", - "description": "Image collection...", - }, - { - "key": "inputs.filePattern", - "title": "Filename pattern: ", - "description": "Use a filename pattern to calculate flatfield information by subsets", - }, - { - "key": "inputs.groupBy", - "title": "Grouping Variables: ", - "description": "Group data together with varying variable values.", - }, - { - "key": "inputs.darkfield", - "title": "Calculate darkfield: ", - "description": "If selected, will generate a darkfield image", - }, - { - "key": "inputs.photobleach", - "title": "Calclate photobleaching offset: ", - "description": "If selected, will generate an offset scalar for each image", - }, - ], -} - -test_dict_load = OrderedDict( - { - "dictionary": { - "name": "BaSiC Flatfield Correction Plugin", - "version": "1.2.7", - "title": "Flatfield correction using BaSiC algorithm.", - "description": "Generates images used for flatfield correction using the BaSiC algorithm.", - "author": "Nick Schaub (nick.schaub@nih.gov)", - "institution": "National Center for the Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/polusai/polus-plugins", - "website": "https://ncats.nih.gov/preclinical/core/informatics", - "citation": 'Peng et al. "A BaSiC tool for background and shading correction of optical microscopy images" Nature Communications (2017)', - "containerId": "polusai/basic-flatfield-correction-plugin:1.2.7", - "inputs": [ - { - "name": "inpDir", - "type": "collection", - "description": "Input image collection.", - "required": True, - }, - { - "name": "filePattern", - "type": "string", - "description": "Filename pattern used to separate images by channel, timepoint, and replicate.", - "required": True, - }, - { - "name": "darkfield", - "type": "boolean", - "description": "Calculate darkfield image.", - "required": True, - }, - { - "name": "photobleach", - "type": "boolean", - "description": "Calculate photobleaching offsets.", - "required": True, - }, - { - "name": "groupBy", - "type": "string", - "description": "Group images together for flatfield by variable.", - "required": False, - }, - ], - "outputs": [ - { - "name": "outDir", - "type": "collection", - "description": "Output data for the plugin", - } - ], - "ui": [ - { - "key": "inputs.inpDir", - "title": "Input image collection: ", - "description": "Image collection...", - }, - { - "key": "inputs.filePattern", - "title": "Filename pattern: ", - "description": "Use a filename pattern to calculate flatfield information by subsets", - }, - { - "key": "inputs.groupBy", - "title": "Grouping Variables: ", - "description": "Group data together with varying variable values.", - }, - { - "key": "inputs.darkfield", - "title": "Calculate darkfield: ", - "description": "If selected, will generate a darkfield image", - }, - { - "key": "inputs.photobleach", - "title": "Calclate photobleaching offset: ", - "description": "If selected, will generate an offset scalar for each image", - }, - ], - }, - "path": RSRC_PATH.joinpath("g1.json"), - } -) - -REPO_PATH = RSRC_PATH.parent.parent -LOCAL_MANIFESTS = list(REPO_PATH.rglob("*plugin.json")) -LOCAL_MANIFESTS = [ - x for x in LOCAL_MANIFESTS if "cookiecutter.project" not in str(x) -] # filter cookiecutter templates -LOCAL_MANIFEST_NAMES = [str(x) for x in LOCAL_MANIFESTS] - - -def _get_path(manifest): - """Return path of local plugin manifest.""" - return PLUGINS[manifest][max(PLUGINS[manifest])] - - -@pytest.mark.repo -@pytest.mark.parametrize("manifest", LOCAL_MANIFESTS, ids=LOCAL_MANIFEST_NAMES) -def test_manifests_local(manifest): - """Test local (repo) manifests.""" - assert isinstance(validate_manifest(manifest), (WIPPPluginManifest, ComputeSchema)) - - -def test_list_plugins(): - """Test `list_plugins()`.""" - o = list(PLUGINS.keys()) - o.sort() - assert o == list_plugins() - - -@pytest.mark.parametrize("manifest", list_plugins(), ids=list_plugins()) -def test_manifests_plugindir(manifest): - """Test manifests available in polus-plugins installation dir.""" - p = _get_path(manifest) - assert isinstance(validate_manifest(p), (WIPPPluginManifest, ComputeSchema)) - - -@pytest.mark.parametrize("type_", test_dict_load.values(), ids=test_dict_load.keys()) -def test_load_manifest(type_): # test path and dict - """Test _load_manifest() for types path and dict.""" - assert _load_manifest(type_) == d_val - - -bad = [f"b{x}.json" for x in [1, 2, 3]] -good = [f"g{x}.json" for x in [1, 2, 3]] - - -@pytest.mark.parametrize("manifest", bad, ids=bad) -def test_bad_manifest(manifest): - """Test bad manifests raise InvalidManifest error.""" - with pytest.raises(InvalidManifestError): - validate_manifest(REPO_PATH.joinpath("tests", "resources", manifest)) - - -@pytest.mark.parametrize("manifest", good, ids=good) -def test_good_manifest(manifest): - """Test different manifests that all should pass validation.""" - p = RSRC_PATH.joinpath(manifest) - assert isinstance(validate_manifest(p), (WIPPPluginManifest, ComputeSchema)) diff --git a/tests/test_plugins.py b/tests/test_plugins.py deleted file mode 100644 index 4a0d60ca1..000000000 --- a/tests/test_plugins.py +++ /dev/null @@ -1,198 +0,0 @@ -# type: ignore -# pylint: disable=C0116, W0621, W0613 -"""Plugin Object Tests.""" -from pathlib import Path - -import pytest - -import polus.plugins as pp -from polus.plugins._plugins.classes import Plugin, _load_plugin - -RSRC_PATH = Path(__file__).parent.joinpath("resources") -OMECONVERTER = RSRC_PATH.joinpath("omeconverter022.json") -BASIC_131 = ( - "https://raw.githubusercontent.com/PolusAI/polus-plugins/" - "e8f23a3661e3e5f7ad7dc92f4b0d9c31e7076589/regression/" - "polus-basic-flatfield-correction-plugin/plugin.json" -) -BASIC_127 = ( - "https://raw.githubusercontent.com/PolusAI/polus-plugins/" - "440e64a51a578e21b574009424a75c848ebbbb03/regression/polus-basic" - "-flatfield-correction-plugin/plugin.json" -) - - -@pytest.fixture -def remove_all(): - """Remove all plugins.""" - pp.remove_all() - - -def test_empty_list(remove_all): - """Test empty list.""" - assert pp.list == [] - - -def test_submit_plugin(remove_all): - """Test submit_plugin.""" - pp.submit_plugin(OMECONVERTER) - assert pp.list == ["OmeConverter"] - - -@pytest.fixture -def submit_omeconverter(): - pp.submit_plugin(OMECONVERTER) - - -@pytest.fixture -def submit_basic131(): - pp.submit_plugin(BASIC_131) - - -@pytest.fixture -def submit_basic127(): - pp.submit_plugin(BASIC_127) - - -def test_get_plugin(submit_omeconverter): - """Test get_plugin.""" - assert isinstance(pp.get_plugin("OmeConverter"), Plugin) - - -def test_url1(submit_omeconverter, submit_basic131): - """Test url submit.""" - assert sorted(pp.list) == ["BasicFlatfieldCorrectionPlugin", "OmeConverter"] - - -def test_url2(submit_omeconverter, submit_basic131, submit_basic127): - """Test url submit.""" - assert sorted(pp.list) == ["BasicFlatfieldCorrectionPlugin", "OmeConverter"] - - -def test_load_plugin(submit_omeconverter): - """Test load_plugin.""" - assert _load_plugin(OMECONVERTER).name == "OME Converter" - - -def test_load_plugin2(submit_basic131): - """Test load_plugin.""" - assert _load_plugin(BASIC_131).name == "BaSiC Flatfield Correction Plugin" - - -def test_attr1(submit_omeconverter): - """Test attributes.""" - p_attr = pp.OmeConverter - p_get = pp.get_plugin("OmeConverter") - for attr in p_get.__dict__: - if attr == "id": - continue - assert getattr(p_attr, attr) == getattr(p_get, attr) - - -def test_attr2(submit_basic131): - """Test attributes.""" - p_attr = pp.BasicFlatfieldCorrectionPlugin - p_get = pp.get_plugin("BasicFlatfieldCorrectionPlugin") - for attr in p_get.__dict__: - if attr == "id": - continue - assert getattr(p_attr, attr) == getattr(p_get, attr) - - -def test_versions(submit_basic131, submit_basic127): - """Test versions.""" - assert sorted(pp.get_plugin("BasicFlatfieldCorrectionPlugin").versions) == [ - "1.2.7", - "1.3.1", - ] - - -def test_get_max_version1(submit_basic131, submit_basic127): - """Test get max version.""" - plug = pp.get_plugin("BasicFlatfieldCorrectionPlugin") - assert plug.version == "1.3.1" - - -def test_get_max_version2(submit_basic131, submit_basic127): - """Test get max version.""" - plug = pp.BasicFlatfieldCorrectionPlugin - assert plug.version == "1.3.1" - - -def test_get_specific_version(submit_basic131, submit_basic127): - """Test get specific version.""" - plug = pp.get_plugin("BasicFlatfieldCorrectionPlugin", "1.2.7") - assert plug.version == "1.2.7" - - -def test_remove_version(submit_basic131, submit_basic127): - """Test remove version.""" - pp.remove_plugin("BasicFlatfieldCorrectionPlugin", "1.2.7") - assert pp.BasicFlatfieldCorrectionPlugin.versions == ["1.3.1"] - - -def test_remove_all_versions_plugin( - submit_basic131, submit_basic127, submit_omeconverter -): - """Test remove all versions plugin.""" - pp.remove_plugin("BasicFlatfieldCorrectionPlugin") - assert pp.list == ["OmeConverter"] - - -def test_submit_str_1(): - """Test submit_plugin with string.""" - pp.remove_all() - pp.submit_plugin(str(OMECONVERTER)) - assert pp.list == ["OmeConverter"] - - -def test_submit_str_2(): - """Test submit_plugin with string.""" - pp.remove_all() - pp.submit_plugin(str(OMECONVERTER.absolute())) - assert pp.list == ["OmeConverter"] - - -@pytest.fixture -def plug1(): - """Configure the class.""" - pp.submit_plugin(BASIC_131) - plug1 = pp.BasicFlatfieldCorrectionPlugin - plug1.inpDir = RSRC_PATH.absolute() - plug1.outDir = RSRC_PATH.absolute() - plug1.filePattern = "*.ome.tif" - plug1.darkfield = True - plug1.photobleach = False - return plug1 - - -@pytest.fixture(scope="session") -def config_path(tmp_path_factory): - """Temp config path.""" - return tmp_path_factory.mktemp("config") / "config1.json" - - -def test_save_load_config(plug1, config_path): - """Test save_config, load_config from config file.""" - plug1.save_config(config_path) - plug2 = pp.load_config(config_path) - for i_o in ["inpDir", "outDir", "filePattern"]: - assert getattr(plug2, i_o) == getattr(plug1, i_o) - assert plug2.id == plug1.id - - -def test_load_config_no_plugin(plug1, config_path): - """Test load_config after removing plugin.""" - plug1.save_config(config_path) - plug1_id = plug1.id - pp.remove_plugin("BasicFlatfieldCorrectionPlugin") - assert pp.list == ["OmeConverter"] - plug2 = pp.load_config(config_path) - assert isinstance(plug2, Plugin) - assert plug2.id == plug1_id - - -def test_remove_all(submit_basic131, submit_basic127, submit_omeconverter): - """Test remove_all.""" - pp.remove_all() - assert pp.list == [] diff --git a/tests/test_version.py b/tests/test_version.py deleted file mode 100644 index 512bdc420..000000000 --- a/tests/test_version.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Test Version object and cast_version utility function.""" -import pydantic -import pytest -from pydantic import ValidationError - -from polus.plugins._plugins.io import Version -from polus.plugins._plugins.utils import cast_version - -PYDANTIC_VERSION = pydantic.__version__.split(".", maxsplit=1)[0] - -GOOD_VERSIONS = [ - "1.2.3", - "1.4.7-rc1", - "4.1.5", - "12.8.3", - "10.2.0", - "2.2.3-dev5", - "0.3.4", - "0.2.34-rc23", -] -BAD_VERSIONS = ["02.2.3", "002.2.3", "1.2", "1.0", "1.03.2", "23.3.03", "d.2.4"] - -PV = PYDANTIC_VERSION -print(PV) - - -@pytest.mark.parametrize("ver", GOOD_VERSIONS, ids=GOOD_VERSIONS) -def test_version(ver): - """Test Version pydantic model.""" - if PV == "1": - assert isinstance(Version(version=ver), Version) - assert isinstance(Version(ver), Version) - - -@pytest.mark.skipif(int(PV) > 1, reason="requires pydantic 1") -@pytest.mark.parametrize("ver", GOOD_VERSIONS, ids=GOOD_VERSIONS) -def test_cast_version(ver): - """Test cast_version utility function.""" - assert isinstance(cast_version(ver), Version) - - -@pytest.mark.parametrize("ver", BAD_VERSIONS, ids=BAD_VERSIONS) -def test_bad_version1(ver): - """Test ValidationError is raised for invalid versions.""" - if PV == "1": - with pytest.raises(ValidationError): - assert isinstance(cast_version(ver), Version) - with pytest.raises(ValidationError): - assert isinstance(Version(ver), Version) - - -MAJOR_VERSION_EQUAL = ["2.4.3", "2.98.28", "2.1.2", "2.0.0", "2.4.0"] -MINOR_VERSION_EQUAL = ["1.3.3", "7.3.4", "98.3.12", "23.3.0", "1.3.5"] -PATCH_EQUAL = ["12.2.7", "2.3.7", "1.7.7", "7.7.7", "7.29.7"] - - -@pytest.mark.parametrize("ver", MAJOR_VERSION_EQUAL, ids=MAJOR_VERSION_EQUAL) -def test_major(ver): - """Test major version.""" - if PV == "2": - assert Version(ver).major == 2 - else: - assert cast_version(ver).major == 2 - - -@pytest.mark.parametrize("ver", MINOR_VERSION_EQUAL, ids=MINOR_VERSION_EQUAL) -def test_minor(ver): - """Test minor version.""" - if PV == "2": - assert Version(ver).minor == 3 - else: - assert cast_version(ver).minor == 3 - - -@pytest.mark.parametrize("ver", PATCH_EQUAL, ids=PATCH_EQUAL) -def test_patch(ver): - """Test patch version.""" - if PV == "2": - assert Version(ver).patch == 7 - else: - assert cast_version(ver).patch == 7 - - -def test_gt1(): - """Test greater than operator.""" - if PV == "2": - assert Version("1.2.3") > Version("1.2.1") - else: - assert cast_version("1.2.3") > cast_version("1.2.1") - - -def test_gt2(): - """Test greater than operator.""" - if PV == "2": - assert Version("5.7.3") > Version("5.6.3") - else: - assert cast_version("5.7.3") > cast_version("5.6.3") - - -def test_st1(): - """Test less than operator.""" - if PV == "2": - assert Version("5.7.3") < Version("5.7.31") - else: - assert cast_version("5.7.3") < cast_version("5.7.31") - - -def test_st2(): - """Test less than operator.""" - if PV == "2": - assert Version("1.0.2") < Version("2.0.2") - else: - assert cast_version("1.0.2") < cast_version("2.0.2") - - -def test_eq1(): - """Test equality operator.""" - if PV == "2": - assert Version("1.3.3") == Version("1.3.3") - else: - assert Version(version="1.3.3") == cast_version("1.3.3") - - -def test_eq2(): - """Test equality operator.""" - if PV == "2": - assert Version("5.4.3") == Version("5.4.3") - else: - assert Version(version="5.4.3") == cast_version("5.4.3") - - -def test_eq3(): - """Test equality operator.""" - if PV == "2": - assert Version("1.3.3") != Version("1.3.8") - else: - assert Version(version="1.3.3") != cast_version("1.3.8") - - -def test_eq_str1(): - """Test equality with str.""" - if PV == "2": - assert Version("1.3.3") == "1.3.3" - else: - assert Version(version="1.3.3") == "1.3.3" - - -def test_lt_str1(): - """Test equality with str.""" - if PV == "2": - assert Version("1.3.3") < "1.5.3" - else: - assert Version(version="1.3.3") < "1.5.3" - - -def test_gt_str1(): - """Test equality with str.""" - if PV == "2": - assert Version("4.5.10") > "4.5.9" - else: - assert Version(version="4.5.10") > "4.5.9" - - -def test_eq_no_str(): - """Test equality with non-string.""" - if PV == "2": - with pytest.raises(TypeError): - assert Version("1.3.3") == 1.3 - else: - with pytest.raises(TypeError): - assert Version(version="1.3.3") == 1.3 diff --git a/to_clt.py b/to_clt.py deleted file mode 100644 index a2dd9b9e3..000000000 --- a/to_clt.py +++ /dev/null @@ -1,108 +0,0 @@ -# ruff: noqa -"""Script to convert all WIPP manifests to CLT. - -This script will first convert all WIPP manifests to ICT and then to CLT. -WIPP -> ICT -> CLT. -""" - -# pylint: disable=W0718, W1203 -import logging -from pathlib import Path - -import typer -from ict import ICT -from tqdm import tqdm - -app = typer.Typer(help="Convert WIPP manifests to ICT.") -ict_logger = logging.getLogger("ict") -fhandler = logging.FileHandler("clt_conversion.log") -fformat = logging.Formatter( - "%(asctime)s - %(levelname)s - %(message)s", datefmt="%m/%d/%Y %I:%M:%S %p" -) -fhandler.setFormatter(fformat) -fhandler.setLevel("INFO") -ict_logger.setLevel("INFO") -ict_logger.addHandler(fhandler) -ict_logger.setLevel(logging.INFO) -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - datefmt="%m/%d/%Y %I:%M:%S %p", -) -logger = logging.getLogger("wipp_to_clt") -logger.addHandler(fhandler) - -REPO_PATH = Path(__file__).parent -LOCAL_MANIFESTS = list(REPO_PATH.rglob("*plugin.json")) -logger.info(f"Found {len(LOCAL_MANIFESTS)} manifests in {REPO_PATH}") -IGNORE_LIST = ["cookiecutter", ".env", "Shared-Memory-OpenMP"] -# Shared-Memory-OpenMP ignored for now until version -# and container are fixed in the manifest -LOCAL_MANIFESTS = [ - x for x in LOCAL_MANIFESTS if not any(ig in str(x) for ig in IGNORE_LIST) -] - - -@app.command() -def main( - all_: bool = typer.Option( - False, - "--all", - "-a", - help="Convert all manifests in the repository.", - ), - name: str = typer.Option( - None, - "--name", - "-n", - help="Name of the plugin to convert.", - ), -) -> None: - """Convert WIPP manifests to ICT.""" - problems = {} - converted = 0 - if not all_ and name is None: - logger.error("Please provide a name if not converting all manifests.") - raise typer.Abort - if name is not None: - if all_: - logger.warning("Ignoring --all flag since a name was provided.") - logger.info(f"name: {name}") - all_ = False - logger.info(f"all: {all_}") - if all_: - n = len(LOCAL_MANIFESTS) - for manifest in tqdm(LOCAL_MANIFESTS): - try: - ict_ = ICT.from_wipp(manifest) - ict_name = ( - ict_.name.split("/")[-1].lower() + ".cwl" # pylint: disable=E1101 - ) - ict_.save_clt(manifest.with_name(ict_name)) - - converted += 1 - - except BaseException as e: - problems[Path(manifest).parts[4:-1]] = str(e) - if name is not None: - n = 1 - for manifest in [x for x in LOCAL_MANIFESTS if name in str(x)]: - try: - ict_ = ICT.from_wipp(manifest) - ict_name = ( - ict_.name.split("/")[-1].lower() + ".cwl" # pylint: disable=E1101 - ) - ict_.save_clt(manifest.with_name(ict_name)) - converted += 1 - - except BaseException as e: - problems[Path(manifest).parts[4:-1]] = str(e) - - logger.info(f"Converted {converted}/{n} plugins") - if len(problems) > 0: - logger.error(f"Problems: {problems}") - logger.info(f"There were {len(problems)} problems in {n} manifests.") - - -if __name__ == "__main__": - app() diff --git a/to_ict.py b/to_ict.py deleted file mode 100644 index fcb858d18..000000000 --- a/to_ict.py +++ /dev/null @@ -1,99 +0,0 @@ -# ruff: noqa -"""Script to convert all WIPP manifests to ICT.""" - -# pylint: disable=W0718, W1203 -import logging -from pathlib import Path - -import typer -from ict import ICT, validate -from tqdm import tqdm - -app = typer.Typer(help="Convert WIPP manifests to ICT.") -ict_logger = logging.getLogger("ict") -fhandler = logging.FileHandler("ict_conversion.log") -fformat = logging.Formatter( - "%(asctime)s - %(levelname)s - %(message)s", datefmt="%m/%d/%Y %I:%M:%S %p" -) -fhandler.setFormatter(fformat) -fhandler.setLevel("INFO") -ict_logger.setLevel("INFO") -ict_logger.addHandler(fhandler) -ict_logger.setLevel(logging.INFO) -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - datefmt="%m/%d/%Y %I:%M:%S %p", -) -logger = logging.getLogger("wipp_to_ict") -logger.addHandler(fhandler) - -REPO_PATH = Path(__file__).parent -LOCAL_MANIFESTS = list(REPO_PATH.rglob("*plugin.json")) -logger.info(f"Found {len(LOCAL_MANIFESTS)} manifests in {REPO_PATH}") -IGNORE_LIST = ["cookiecutter", ".env", "Shared-Memory-OpenMP"] -# Shared-Memory-OpenMP ignored for now until version -# and container are fixed in the manifest -LOCAL_MANIFESTS = [ - x for x in LOCAL_MANIFESTS if not any(ig in str(x) for ig in IGNORE_LIST) -] - - -@app.command() -def main( - all_: bool = typer.Option( - False, - "--all", - "-a", - help="Convert all manifests in the repository.", - ), - name: str = typer.Option( - None, - "--name", - "-n", - help="Name of the plugin to convert.", - ), -) -> None: - """Convert WIPP manifests to ICT.""" - problems = {} - converted = 0 - if not all_ and name is None: - logger.error("Please provide a name if not converting all manifests.") - raise typer.Abort - if name is not None: - if all_: - logger.warning("Ignoring --all flag since a name was provided.") - logger.info(f"name: {name}") - all_ = False - logger.info(f"all: {all_}") - if all_: - n = len(LOCAL_MANIFESTS) - for manifest in tqdm(LOCAL_MANIFESTS): - try: - ict_ = ICT.from_wipp(manifest) - yaml_path = ict_.save_yaml(manifest.with_name("ict.yaml")) - validate(yaml_path) - converted += 1 - - except BaseException as e: - problems[Path(manifest).parts[4:-1]] = str(e) - if name is not None: - n = 1 - for manifest in [x for x in LOCAL_MANIFESTS if name in str(x)]: - try: - ict_ = ICT.from_wipp(manifest) - yaml_path = ict_.save_yaml(manifest.with_name("ict.yaml")) - validate(yaml_path) - converted += 1 - - except BaseException as e: - problems[Path(manifest).parts[4:-1]] = str(e) - - logger.info(f"Converted {converted}/{n} plugins") - if len(problems) > 0: - logger.error(f"Problems: {problems}") - logger.info(f"There were {len(problems)} problems in {n} manifests.") - - -if __name__ == "__main__": - app()