From bf0885d7ac85b05744d550b2f2ff7ad473c0d90f Mon Sep 17 00:00:00 2001 From: agerardin Date: Thu, 25 Jan 2024 11:53:13 -0500 Subject: [PATCH] Update: create plugin skeletons following the updated plugin standards. (#435) * Update: create plugin skeletons following the updated plugin standards. * update: polus-template now create packages in polus.plugins. * update: update to new standards. * filepattern updated to 2.0.0 * package is installed through poetry so now the tests reference only the package. * update: new standard updates * better management of package declarations * use typing * add preview option * refactored main, add argument checks * added tests fixtures and test of the client * better management of versions * update dependencies * Fix : review plugin generation logic. * fix: added logging info and fix bug when no intermdiary directories are created. * fix: correct typo. * chore: update plugin scaffold to new standards. * fix: bumpversion config file is now templated * update: - improve tests -clean up hooks. * --amend * --amend * update: updates after reviews. --------- Co-authored-by: Nicholas-Schaub --- .gitignore | 1 + utils/polus-python-template/.bumpversion.cfg | 23 +++ utils/polus-python-template/.gitignore | 1 + utils/polus-python-template/CHANGELOG.md | 9 + utils/polus-python-template/README.md | 176 ++++++++---------- utils/polus-python-template/VERSION | 2 +- utils/polus-python-template/bfio.cfg | 16 -- utils/polus-python-template/cookiecutter.json | 47 ++--- utils/polus-python-template/filepattern.cfg | 12 -- .../hooks/post_gen_project.py | 63 +++++++ .../hooks/pre_gen_project.py | 55 ++++++ utils/polus-python-template/pyproject.toml | 32 ++++ utils/polus-python-template/requirements.txt | 4 - .../.bumpversion.cfg | 29 +++ .../.dockerignore | 4 + .../.gitignore | 1 + .../CHANGELOG.md | 5 + .../Dockerfile | 26 +++ .../{{cookiecutter.container_name}}/README.md | 23 +++ .../{{cookiecutter.container_name}}/VERSION | 1 + .../build-docker.sh | 4 + .../plugin.json | 63 +++++++ .../pyproject.toml | 32 ++++ .../run-plugin.sh | 20 ++ .../__init__.py | 7 + .../__main__.py | 87 +++++++++ .../{{ cookiecutter.package_name }}.py | 16 ++ .../tests/__init__.py | 1 + .../tests/conftest.py | 147 +++++++++++++++ .../tests/test_cli.py | 96 ++++++++++ .../test_{{cookiecutter.package_name}}.py | 22 +++ .../{{cookiecutter.project_slug}}/Dockerfile | 19 -- .../{{cookiecutter.project_slug}}/README.md | 32 ---- .../{{cookiecutter.project_slug}}/VERSION | 1 - .../build-docker.sh | 4 - .../bumpversion.cfg | 10 - .../package-release.sh | 16 -- .../{{cookiecutter.project_slug}}/plugin.json | 34 ---- .../run-plugin.sh | 36 ---- .../{{cookiecutter.project_slug}}/src/main.py | 165 ---------------- .../src/requirements.txt | 2 - .../tests/__init__.py | 13 -- .../tests/plugin_test.py | 12 -- .../tests/version_test.py | 42 ----- 44 files changed, 856 insertions(+), 555 deletions(-) create mode 100644 utils/polus-python-template/.bumpversion.cfg create mode 100644 utils/polus-python-template/.gitignore create mode 100644 utils/polus-python-template/CHANGELOG.md delete mode 100644 utils/polus-python-template/bfio.cfg delete mode 100644 utils/polus-python-template/filepattern.cfg create mode 100644 utils/polus-python-template/hooks/post_gen_project.py create mode 100644 utils/polus-python-template/hooks/pre_gen_project.py create mode 100644 utils/polus-python-template/pyproject.toml delete mode 100644 utils/polus-python-template/requirements.txt create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/.bumpversion.cfg create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/.dockerignore create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/.gitignore create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/CHANGELOG.md create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/Dockerfile create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/README.md create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/VERSION create mode 100755 utils/polus-python-template/{{cookiecutter.container_name}}/build-docker.sh create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/plugin.json create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/pyproject.toml create mode 100755 utils/polus-python-template/{{cookiecutter.container_name}}/run-plugin.sh create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/src/{{cookiecutter.package_folders}}/__init__.py create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/src/{{cookiecutter.package_folders}}/__main__.py create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/src/{{cookiecutter.package_folders}}/{{ cookiecutter.package_name }}.py create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/tests/__init__.py create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/tests/conftest.py create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/tests/test_cli.py create mode 100644 utils/polus-python-template/{{cookiecutter.container_name}}/tests/test_{{cookiecutter.package_name}}.py delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/Dockerfile delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/README.md delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/VERSION delete mode 100755 utils/polus-python-template/{{cookiecutter.project_slug}}/build-docker.sh delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/bumpversion.cfg delete mode 100755 utils/polus-python-template/{{cookiecutter.project_slug}}/package-release.sh delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/plugin.json delete mode 100755 utils/polus-python-template/{{cookiecutter.project_slug}}/run-plugin.sh delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/src/main.py delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/src/requirements.txt delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/tests/__init__.py delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/tests/plugin_test.py delete mode 100644 utils/polus-python-template/{{cookiecutter.project_slug}}/tests/version_test.py diff --git a/.gitignore b/.gitignore index 8d7a62dd7..a07072cae 100644 --- a/.gitignore +++ b/.gitignore @@ -167,6 +167,7 @@ data # local manifests src/polus/plugins/_plugins/manifests/* + # allow python scripts inside manifests dir !src/polus/plugins/_plugins/manifests/*.py diff --git a/utils/polus-python-template/.bumpversion.cfg b/utils/polus-python-template/.bumpversion.cfg new file mode 100644 index 000000000..cdd5e56ac --- /dev/null +++ b/utils/polus-python-template/.bumpversion.cfg @@ -0,0 +1,23 @@ +[bumpversion] +current_version = 1.1.0-dev0 +commit = False +tag = False +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? +serialize = + {major}.{minor}.{patch}-{release}{dev} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = _ +first_value = dev +values = + dev + _ + +[bumpversion:part:dev] + +[bumpversion:file:pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" + +[bumpversion:file:README.md] diff --git a/utils/polus-python-template/.gitignore b/utils/polus-python-template/.gitignore new file mode 100644 index 000000000..d27abdcf4 --- /dev/null +++ b/utils/polus-python-template/.gitignore @@ -0,0 +1 @@ +poetry.lock \ No newline at end of file diff --git a/utils/polus-python-template/CHANGELOG.md b/utils/polus-python-template/CHANGELOG.md new file mode 100644 index 000000000..489dc11b7 --- /dev/null +++ b/utils/polus-python-template/CHANGELOG.md @@ -0,0 +1,9 @@ +# CHANGELOG + +# 1.0.0 + +* Generate plugins from templates using cookiecutter. + +# 1.1.0 + +* Generate plugins following updated [standard guidelines](https://labshare.atlassian.net/wiki/spaces/WIPP/pages/3275980801/Python+Plugin+Standards) diff --git a/utils/polus-python-template/README.md b/utils/polus-python-template/README.md index 3c64bdc02..021868354 100644 --- a/utils/polus-python-template/README.md +++ b/utils/polus-python-template/README.md @@ -1,140 +1,112 @@ -# WIPP Plugin Cookie Cutter (for Python) +# WIPP Plugin Cookie Cutter (for Python) (1.1.0-dev0) -This repository is a cookie cutter template that gives the basic structure of a -WIPP plugin, and it's specially tailored to Python and Linux. However, even if -the code base for the plugin is not Python, it is still useful for generating -a basic skeleton of a plugin. +This repository is a cookie cutter template that creates the basic scaffold structure of a +polus plugin and add it to the polus plugins directory structure. ## How to use - 1. Clone `polus-plugins` and change to the polus-plugins directory -2. Install the requirements: `pip install -r ./utils/polus-python-template/requirements.txt` -3. Ignore changes to `cookiecutter.json` using: `git update-index --assume-unchanged ./utils/polus-python-template/cookiecutter.json` -4. Modify `cookiecutter.json` to include author and plugin information. -5. Create your plugin skeleton: `cookiecutter ./utils/polus-python-template/ --no-input` +2. `cd /utils/polus-python-template/` +3. (optional) Install poetry if not available. +4. (optional) Create a dedicated environment with conda or venv. +5. Install the dependencies: `poetry install` +6. Ignore changes to `cookiecutter.json` using: `git update-index --assume-unchanged cookiecutter.json` +7. Modify `cookiecutter.json` to include author and plugin information.`plugin_package` should always start with `polus.plugins`. +** NOTE: ** Do not edit values in brackets ({}) as they are edited by cookiecutter directly. +Those are automatically generated from the previous entries. If your plugin is called +"Awesome Function", then the plugin folder and docker container will have the name `awesome-function-plugin`. +8. Create your plugin skeleton: ` python -m cookiecutter . --no-input` -** NOTE: ** Do not modify `project_slug`. This is automatically generated from -the name of the plugin. If your plugin is called "Awesome Segmentation", then -the plugin folder and docker container will have the name -`polus-awesome-segmentation-plugin`. -## What's New? +## Plugin Standard +The generated plugin will be compatible with polus most up-to-date guidelines : +see [standard guidelines](https://labshare.atlassian.net/wiki/spaces/WIPP/pages/3275980801/Python+Plugin+Standards) -## bump2version +The code generated provides out-of-box support for : + - customizing the plugin code. + - implementing tests. + - creating and running a container. + - managing versioning. + - updating documentation (README, CHANGELOG). + - maintaining a WIPP manifest (plugin.json). -Making sure that the file version is consistent across files in a plugin can be -challenging, so the Python template now uses -[bump2version](https://github.com/c4urself/bump2version) -to help manage versioning. This automatically changes the `VERSION` and -`plugin.json` files to the next version, preventing you from having to remember -to change the version in all 3 places. The `bumpversion.cfg` can be modified to -change the version in other files as well. -To use this feature: -1. `pip install bump2version` -2. `bump2version --config-file bumpversion.cfg` -3. If an error is thrown about uncommited work, use the `--alow-dirty` option +## Executing the plugin -## unittest +The plugin should be run as a package. +To install the package : -A simple `unittest` has been added to the template that verifies the `VERSION` -matches the `plugin.json` version information. Before submitting a PR to -`polus-plugins`, other unit tests should be created and added to the `tests` -directory. The test classes that are created should be added to the tuple -`test_cases` in `tests/__init__.py`. +`pip install .` -## run-plugin.sh +The skeleton code can be run this way : +From the plugin's top directory (with the default values): -A new shell script (`run-plugin.sh`) has been added to help setup testing of -docker containers locally. It comes configured with the proper `docker run` -command already set up so that input variables just need to be defined. +`python -m polus.plugins1.package1.package2.awesome_function -i /tmp/inp -o /tmp/out` -## Explanation of File Structure and Settings +This should print some logs with the provided inputs and outputs and return. -### General Structure +## Running tests +Plugin's developer should use `pytest`. +Some simple tests have been added to the template as examples. +Before submitting a PR to `polus-plugins`, other unit tests should be created and added to the `tests` +directory. -In general, the structure of a plugin should have the following files as a minimum: +To run tests : -``` -plugin-root/ - - VERSION* - - build-docker.sh - - run-plugin.sh - - bumpversion.cfg - - Dockerfile* - - README.md* - - plugin.json* - - src/ - - main.py - - requirements.txt - - tests/ - - __init__.py - - version_test.py -``` +From the plugin's top directory, type `python -m pytest`. +Depending on how you have set up your environment, you may be able to run the pytest cli directly `pytest`. See pytest doc for how the project source directory is scanned to collect tests. +This should run a test successfully and return. -Files with a `*` at the end indicate files that are necessary. If none of the -other files are modified, there are some built in tools to simplify -containerization and deployment of existing code to WIPP. -### VERSION +## Creating and running a container -This indicates the version of the plugin. It should follow the -[semantic versioning](https://semver.org/) standard (for example `2.0.0`). The -only thing that should be in this file is the version. The cookie cutter -template defaults to `0.1.0`. +` ./build-docker.sh && ./run-plugin.sh` -This file is used to tag the docker container built with the `build-docker.sh` -script and by Jenkins if the plugin is merged into `labshare/polus-plugins`. +Build the docker image and run the container. -### Dockerfile +### DockerFile +A docker image is build from a base image with common dependencies pre-installed. +The image entrypoint will run the plugin's package entrypoint. -This is a basic dockerfile. In general, this should contain all the necessary -tools to run a basic Python plugin. +### build-docker.sh +Run this script to build the container. -The Dockerfile uses Debian Linux (slim buster) with Python 3.9 installed on it -as the base image ([python:3.9-slim](https://hub.docker.com/_/python)). +### run-plugin.sh +Run the container locally. -If `use_bfio` is set to true, then `labshare/polus-bfio-util` is used as the -base image. -If `use_java` is set to true, then the container uses a version of -`labshare/polus-bfio-util` that comes with Java, javabridge, and bioformats. +## Customize the plugin -If `use_filepattern` is set to true, then `requirements.txt` will be generated -to include `filepattern` and `main.py` will have some boilerplate code for -using the `filepattern` package. +### Project code -For more information, check out the repositories for -[javabridge](https://github.com/LeeKamentsky/python-javabridge), -[python-bioformats](https://github.com/CellProfiler/python-bioformats), -and [bfio](https://github.com/Nicholas-Schaub/polus-plugins/tree/master/utils/polus-bfio-util). +A set of common dependencies are added to `pyproject.toml`. +Update according to your needs. -### README.md +### Managing versioning -A basic description of what the plugin does. This should define all the inputs -and outputs. Cookiecutter should autogenerate the input and output table, but -double check to make sure. +Making sure that the file version is consistent across files in a plugin can be +challenging, so the Python template now uses +[bump2version](https://github.com/c4urself/bump2version) +to help manage versioning. This automatically changes the `VERSION` and +`plugin.json` files to the next version, preventing you from having to remember +to change the version everywhere. The `bumpversion.cfg` can be modified to +change the version in other files as well. -### plugin.json +To use this feature: +`bump2version --config-file bumpversion.cfg` -This file defines the input and output variables for WIPP, and defines the UI -components showed to the user. This should be automatically generated for basic -variable types, but may need to be modified to work properly. +### Documentation -### build-docker.sh +#### README.md -This file builds a docker container using the name of the plugin and a tag using -`VERSION`. +A basic description of what the plugin does. This should define all the inputs +and outputs. -### src/main.py +#### CHANGELOG.md -This is the file called from the commandline from the docker container. Cookie -cutter autogenerates some basic code based on the inputs specified in the -cookiecutter json, for example there is code to parse commandline arguments. If -the name of this file is changed, then the `Dockerfile` will need to be modified -with the name of the new file. +Documents updates made to the plugin. -### src/requirements.txt -This file should contain a list of the packages (including versions) that are -used by the plugin. It is important to make this as simple as possible. +### WIPP manifest (plugin.json). + +This file defines the input and output variables for WIPP, and defines the UI +components showed to the user. diff --git a/utils/polus-python-template/VERSION b/utils/polus-python-template/VERSION index afaf360d3..3018fdcd4 100644 --- a/utils/polus-python-template/VERSION +++ b/utils/polus-python-template/VERSION @@ -1 +1 @@ -1.0.0 \ No newline at end of file +1.1.0-dev0 diff --git a/utils/polus-python-template/bfio.cfg b/utils/polus-python-template/bfio.cfg deleted file mode 100644 index 8d318019f..000000000 --- a/utils/polus-python-template/bfio.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[bumpversion] -current_version = 2.1.9 -commit = False -tag = False - -[bumpversion:file:{{cookiecutter.project_slug}}/Dockerfile] -search = labshare/polus-bfio-util:{current_version} -replace = labshare/polus-bfio-util:{new_version} - -[bumpversion:file:{{cookiecutter.project_slug}}/src/requirements.txt] -search = bfio[all]=={current_version} -replace = bfio[all]=={new_version} - -[bumpversion:file:requirements.txt] -search = bfio[all]=={current_version} -replace = bfio[all]=={new_version} diff --git a/utils/polus-python-template/cookiecutter.json b/utils/polus-python-template/cookiecutter.json index 8bd47c8fe..030f8cf3c 100644 --- a/utils/polus-python-template/cookiecutter.json +++ b/utils/polus-python-template/cookiecutter.json @@ -1,37 +1,16 @@ { "author": "Data Scientist", - "email": "data.scientist@labshare.org", - "github_username": "datascientist", - "project_name": "WIPP Widget", - "project_short_description": "Everything you need to start a WIPP plugin.", - "version": "0.1.0", - "use_bfio": true, - "use_filepattern": true, - "use_java": false, - "_inputs": { - "inpDir": { - "type": "collection", - "title": "Input collection", - "description": "Input image collection to be processed by this plugin", - "required": "True" - }, - "filePattern": { - "type": "string", - "title": "Filename pattern", - "description": "Filename pattern used to separate data", - "required": "False" - } - }, - "_outputs": { - "outDir": { - "type": "collection", - "description": "Output collection" - } - }, - - "project_slug": "polus-{{ cookiecutter.project_name|lower|replace(' ', '-') }}-plugin", + "author_email": "data.scientist@labshare.org", + "plugin_name": "Awesome Plugin", + "plugin_package": "polus.plugins.package1.package2.awesome_function", + "plugin_description": "An awesome function.", + "plugin_version": "0.1.0", - "_extensions": [ - "jinja2_ospath.extensions.OSPathExtension" - ] -} \ No newline at end of file + "package_folders": "{%set folders = cookiecutter.plugin_package.replace('.', '/') %}{{folders}}", + "package_name": "{% set packages = cookiecutter.plugin_package.split('.') %}{{ packages | last }}", + "project_name": "{% set project_name = cookiecutter.plugin_package.replace('_', '-').replace('.', '-') %}{{ project_name }}", + "plugin_slug": "{% set plugin_slug = cookiecutter.package_name.replace('_', '-') %}polus-{{plugin_slug}}-plugin", + "container_name": "{% set container_name = ('-').join(cookiecutter.plugin_slug.split('-')[1:])%}{{ container_name }}", + "container_id": "polusai/{{cookiecutter.container_name}}", + "container_version": "{{cookiecutter.plugin_version}}" +} diff --git a/utils/polus-python-template/filepattern.cfg b/utils/polus-python-template/filepattern.cfg deleted file mode 100644 index 4e39c0132..000000000 --- a/utils/polus-python-template/filepattern.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[bumpversion] -current_version = 1.4.6 -commit = False -tag = False - -[bumpversion:file:{{cookiecutter.project_slug}}/src/requirements.txt] -search = filepattern=={current_version} -replace = filepattern=={new_version} - -[bumpversion:file:requirements.txt] -search = filepattern=={current_version} -replace = filepattern=={new_version} diff --git a/utils/polus-python-template/hooks/post_gen_project.py b/utils/polus-python-template/hooks/post_gen_project.py new file mode 100644 index 000000000..f3f0ee429 --- /dev/null +++ b/utils/polus-python-template/hooks/post_gen_project.py @@ -0,0 +1,63 @@ +import os +import shutil +from pathlib import Path +import logging +from os import environ + +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +POLUS_LOG = getattr(logging, environ.get("POLUS_LOG", "DEBUG")) +logger = logging.getLogger("polus-python-template-post") +logger.setLevel(POLUS_LOG) + + +def create_repository_directories(source_dir): + """ Buid the correct directories inside polus-plugins. + The directory structure must conforms to the plugin's spec : + - dash-separated word in identifier. + - folder hierarchy matches package namespace minus "polus.plugins" + - plugin's folder name reflects the plugin package name but ends with "-plugin" + Ex: polus.plugins.package1.package2.awesome_function becomes + package1/package2/awesome-function-plugin + """ + + # try to find the project's root, otherwise we stay in the + # staging directory + final_dir = source_dir.parent + for folder in Path(final_dir).parent.parents: + if os.path.exists(folder / ".git"): + final_dir = folder + break + + # by default we create a plugin directory at the root + target_dir = final_dir + + # figure out if additional directories need to be created at the root + # make sure we replace underscores + new_dirs = "{{cookiecutter.plugin_package}}".replace("_", "-") + new_dirs = new_dirs.split(".") + # remove polus.plugins so we only keep intermediary directories + # Ex: polus.plugins.package1.package2.awesome_function creates + # package1/package2/ + new_dirs = new_dirs[2:-1] + if len(new_dirs) != 0: + package_dir = os.path.join(*new_dirs) + target_dir = final_dir / package_dir + + # create the plugin directory + os.makedirs(target_dir, exist_ok=True) + + return target_dir + + +def move_project_source_to_final_location(): + """Move staged files to the the final target repo.""" + source_dir = Path(os.getcwd()) + target_dir = create_repository_directories(source_dir) + logger.debug(f"moving sources from {source_dir} to {target_dir}") + shutil.move(source_dir, target_dir) + +# NOTE do not create folder structure with the repo at the moment. +# move_project_source_to_final_location() \ No newline at end of file diff --git a/utils/polus-python-template/hooks/pre_gen_project.py b/utils/polus-python-template/hooks/pre_gen_project.py new file mode 100644 index 000000000..802f5d154 --- /dev/null +++ b/utils/polus-python-template/hooks/pre_gen_project.py @@ -0,0 +1,55 @@ +""" +Validate of template variables before templating the project +""" +import logging +from os import environ + +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +POLUS_LOG = getattr(logging, environ.get("POLUS_LOG", "DEBUG")) +logger = logging.getLogger("polus-python-template-pre") +logger.setLevel(POLUS_LOG) + +# NOTE Those validation could be performed on a plugin.json +# using polus plugins pydantic models. + +author = "{{ cookiecutter.author }}" +# TODO check valid + +author_email = "{{ cookiecutter.author_email }}" +## TODO check valid + +plugin_package = "{{ cookiecutter.plugin_package }}" +if not plugin_package.startswith("polus.plugins."): + raise ValueError( + f"plugin package must be a child of polus.plugins." + + f"plugin_package must start with 'polus.plugins'. Got : {plugin_package}" + ) +if plugin_package.endswith("_plugin"): + raise ValueError( + f"plugin_package must not ends with _plugin. Got : {plugin_package}" + ) + +# TODO check we have a valid python package name + +plugin_version = "{{ cookiecutter.plugin_version }}" +# TODO check version is valid + +project_name = "{{ cookiecutter.project_name }}" +assert not ("_" in project_name) and not ("." in project_name) + +plugin_slug = "{{ cookiecutter.plugin_slug }}" +assert plugin_slug.startswith("polus-") and plugin_slug.endswith("-plugin") + +container_name = "{{ cookiecutter.container_name }}" +assert container_name.endswith("-plugin") + +container_id = "{{ cookiecutter.container_id }}" +assert container_id.startswith("polusai/") + +container_version = "{{ cookiecutter.container_version }}" +assert container_version == plugin_version + +logger.debug(f"plugin_package: {plugin_package}" ) diff --git a/utils/polus-python-template/pyproject.toml b/utils/polus-python-template/pyproject.toml new file mode 100644 index 000000000..005203749 --- /dev/null +++ b/utils/polus-python-template/pyproject.toml @@ -0,0 +1,32 @@ +[tool.poetry] +name = "polus-python-template" +version = "1.1.0-dev0" +description = "" +authors = ["Nick Schaub ", "Antoine Gerardin "] +readme = "README.md" +packages = [{include = "polus_python_template"}] + + +[tool.poetry.dependencies] +python = ">=3.9,<3.12" + +[tool.poetry.group.dev.dependencies] +cookiecutter = "1.7.2" +jinja2_ospath = "0.3.0" +bump2version = "^1.0.1" +pytest = "^7.4" +pytest-sugar = "^0.9.6" +pre-commit = "^3.2.1" +black = "^23.3.0" +mypy = "^1.1.1" +ruff = "^0.0.270" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + + +[tool.pytest.ini_options] +pythonpath = [ + "." +] diff --git a/utils/polus-python-template/requirements.txt b/utils/polus-python-template/requirements.txt deleted file mode 100644 index 7bcb64a9a..000000000 --- a/utils/polus-python-template/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -cookiecutter==1.7.2 -jinja2_ospath==0.3.0 -bfio[all]==2.1.9 -filepattern==1.4.6 \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/.bumpversion.cfg b/utils/polus-python-template/{{cookiecutter.container_name}}/.bumpversion.cfg new file mode 100644 index 000000000..ae20e5c43 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/.bumpversion.cfg @@ -0,0 +1,29 @@ +[bumpversion] +current_version = {{ cookiecutter.plugin_version }} +commit = False +tag = False +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? +serialize = + {major}.{minor}.{patch}-{release}{dev} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = _ +first_value = dev +values = + dev + _ + +[bumpversion:part:dev] + +[bumpversion:file:pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" + +[bumpversion:file:VERSION] + +[bumpversion:file:README.md] + +[bumpversion:file:plugin.json] + +[bumpversion:file:src/{{cookiecutter.package_folders}}/__init__.py] diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/.dockerignore b/utils/polus-python-template/{{cookiecutter.container_name}}/.dockerignore new file mode 100644 index 000000000..7c603f814 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/.dockerignore @@ -0,0 +1,4 @@ +.venv +out +tests +__pycache__ diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/.gitignore b/utils/polus-python-template/{{cookiecutter.container_name}}/.gitignore new file mode 100644 index 000000000..c04bc49f7 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/.gitignore @@ -0,0 +1 @@ +poetry.lock diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/CHANGELOG.md b/utils/polus-python-template/{{cookiecutter.container_name}}/CHANGELOG.md new file mode 100644 index 000000000..ca292da60 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/CHANGELOG.md @@ -0,0 +1,5 @@ +# CHANGELOG + +## {{cookiecutter.container_version}} + +Initial release. diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/Dockerfile b/utils/polus-python-template/{{cookiecutter.container_name}}/Dockerfile new file mode 100644 index 000000000..dc889b03d --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/Dockerfile @@ -0,0 +1,26 @@ +FROM polusai/bfio:2.1.9 + +# environment variables defined in polusai/bfio +# ENV EXEC_DIR="/opt/executables" +# ENV DATA_DIR="/data" +# ENV POLUS_EXT=".ome.tif" +# Change to WARNING for fewer logs, and DEBUG for debugging +ENV POLUS_LOG="INFO" + +ENV POLUS_IMG_EXT=".ome.tif" +ENV POLUS_TAB_EXT=".csv" + +# Work directory defined in the base container +# WORKDIR ${EXEC_DIR} + +COPY pyproject.toml ${EXEC_DIR} +COPY VERSION ${EXEC_DIR} +COPY README.md ${EXEC_DIR} +COPY CHANGELOG.md ${EXEC_DIR} +COPY src ${EXEC_DIR}/src + +RUN pip3 install ${EXEC_DIR} --no-cache-dir + +# Default command. Additional arguments are provided through the command line +ENTRYPOINT ["python3", "-m", "{{cookiecutter.plugin_package}}"] +CMD ["--help"] diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/README.md b/utils/polus-python-template/{{cookiecutter.container_name}}/README.md new file mode 100644 index 000000000..f99b4a8c5 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/README.md @@ -0,0 +1,23 @@ +# {{cookiecutter.plugin_name}} ({{cookiecutter.plugin_version}}) + +{{cookiecutter.plugin_description}} + +## Building + +To build the Docker image for the conversion plugin, run `./build-docker.sh`. + +## Install WIPP Plugin + +If WIPP is running, navigate to the plugins page and add a new plugin. Paste the +contents of `plugin.json` into the pop-up window and submit. + +## Options + +This plugin takes 2 input arguments and 1 output argument: + +| Name | Description | I/O | Type | Default +|---------------|-------------------------|--------|--------| +| inpDir | Input image collection to be processed by this plugin | Input | collection +| filePattern | Filename pattern used to separate data | Input | string | .* +| preview | Generate an output preview | Input | boolean | False +| outDir | Output collection | Output | collection diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/VERSION b/utils/polus-python-template/{{cookiecutter.container_name}}/VERSION new file mode 100644 index 000000000..6c21993e6 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/VERSION @@ -0,0 +1 @@ +{{ cookiecutter.plugin_version }} diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/build-docker.sh b/utils/polus-python-template/{{cookiecutter.container_name}}/build-docker.sh new file mode 100755 index 000000000..cf00ccca2 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/build-docker.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +version=$("] +readme = "README.md" +packages = [{include = "polus", from = "src"}] + +[tool.poetry.dependencies] +python = ">=3.9,<3.12" +bfio = {version = ">=2.3.3,<3.0", extras = ["all"]} +filepattern = ">=2.0.4,<3.0" +preadator = "^0.4.0.dev2" +typer = "^0.7.0" + +[tool.poetry.group.dev.dependencies] +bump2version = "^1.0.1" +pytest = "^7.4" +pytest-sugar = "^0.9.6" +pre-commit = "^3.2.1" +black = "^23.3.0" +mypy = "^1.1.1" +ruff = "^0.0.270" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +pythonpath = [ + "." +] diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/run-plugin.sh b/utils/polus-python-template/{{cookiecutter.container_name}}/run-plugin.sh new file mode 100755 index 000000000..d979d0714 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/run-plugin.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +version=$( None: + """Generate preview of the plugin outputs.""" + + preview = {} + + with Path.open(out_dir / "preview.json", "w") as fw: + json.dump(preview, fw, indent=2) + +@app.command() +def main( + inp_dir: Path = typer.Option( + ..., + "--inpDir", + "-i", + help="Input directory to be processed.", + exists=True, + readable=True, + file_okay=False, + resolve_path=True, + ), + filepattern: str = typer.Option( + ".*", + "--filePattern", + "-f", + help="Filepattern used to filter inputs.", + ), + out_dir: Path = typer.Option( + ..., + "--outDir", + "-o", + help="Output directory.", + exists=True, + writable=True, + file_okay=False, + resolve_path=True, + ), + preview: bool = typer.Option( + False, + "--preview", + "-v", + help="Preview of expected outputs (dry-run)", + show_default=False, + ), +): + """{{cookiecutter.plugin_name}}.""" + logger.info(f"inpDir: {inp_dir}") + logger.info(f"filePattern: {filepattern}") + logger.info(f"outDir: {out_dir}") + + if preview: + generate_preview(inp_dir, out_dir) + logger.info(f"generating preview data in : {out_dir}.") + return + + {{cookiecutter.package_name}}(inp_dir, filepattern, out_dir) + + +if __name__ == "__main__": + app() diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/src/{{cookiecutter.package_folders}}/{{ cookiecutter.package_name }}.py b/utils/polus-python-template/{{cookiecutter.container_name}}/src/{{cookiecutter.package_folders}}/{{ cookiecutter.package_name }}.py new file mode 100644 index 000000000..2573a729b --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/src/{{cookiecutter.package_folders}}/{{ cookiecutter.package_name }}.py @@ -0,0 +1,16 @@ +"""{{ cookiecutter.plugin_name }}.""" + +from pathlib import Path + + +def {{cookiecutter.package_name}}(inp_dir: Path, filepattern: str, out_dir: Path): + """{{cookiecutter.plugin_name}}. + + Args: + inp_dir: input directory to process + filepattern: filepattern to filter inputs + out_dir: output directory + Returns: + None + """ + pass \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/tests/__init__.py b/utils/polus-python-template/{{cookiecutter.container_name}}/tests/__init__.py new file mode 100644 index 000000000..28371ef28 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for {{cookiecutter.package_name}}.""" diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/tests/conftest.py b/utils/polus-python-template/{{cookiecutter.container_name}}/tests/conftest.py new file mode 100644 index 000000000..fd0c32168 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/tests/conftest.py @@ -0,0 +1,147 @@ +"""Test fixtures. + +Set up all data used in tests. +""" +import tempfile +import shutil +from pathlib import Path +import numpy as np +import pytest +import itertools + +from bfio import BioWriter, BioReader + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add options to pytest.""" + parser.addoption( + "--downloads", + action="store_true", + dest="downloads", + default=False, + help="run tests that download large data files", + ) + parser.addoption( + "--slow", + action="store_true", + dest="slow", + default=False, + help="run slow tests", + ) + + + + +IMAGE_SIZES = [(1024 * (2**i) ,1024 * (2**i)) for i in range(1, 2)] +LARGE_IMAGE_SIZES = [(1024 * (2**i) ,1024 * (2**i)) for i in range(4, 5)] +PIXEL_TYPES = [np.uint8, float] +PARAMS = [ + (image_size, pixel_type) + for image_size, pixel_type in itertools.product( + IMAGE_SIZES, PIXEL_TYPES + ) +] +LARGE_DATASET_PARAMS = [ + (image_size, pixel_type) + for image_size, pixel_type in itertools.product( + LARGE_IMAGE_SIZES, PIXEL_TYPES + ) +] + + +FixtureReturnType = tuple[ + Path, # input dir + Path, # output dir + Path, # ground truth path + Path, # input image path + Path, # ground truth path +] + + +@pytest.fixture(params=PARAMS) +def generate_test_data(request: pytest.FixtureRequest) -> FixtureReturnType: + """Generate staging temporary directories with test data and ground truth.""" + + # collect test params + image_size, pixel_type = request.param + test_data = _generate_test_data(image_size, pixel_type) + print(test_data) + yield from test_data + + +@pytest.fixture(params=LARGE_DATASET_PARAMS) +def generate_large_test_data(request: pytest.FixtureRequest) -> FixtureReturnType: + """Generate staging temporary directories with test data and ground truth.""" + + # collect test params + image_size, pixel_type = request.param + test_data =_generate_test_data(image_size, pixel_type) + + print(test_data) + + yield from test_data + + +def _generate_test_data(image_size : tuple[int,int], pixel_type: int) -> FixtureReturnType: + """Generate staging temporary directories with test data and ground truth.""" + + image_x, image_y = image_size + + # staging area + data_dir = Path(tempfile.mkdtemp(suffix="_data_dir")) + inp_dir = data_dir.joinpath("inp_dir") + inp_dir.mkdir(exist_ok=True) + out_dir = data_dir.joinpath("out_dir") + out_dir.mkdir(exist_ok=True) + ground_truth_dir = data_dir.joinpath("ground_truth_dir") + ground_truth_dir.mkdir(exist_ok=True) + + # generate image and ground_truth + img_path = inp_dir.joinpath("img.ome.tif") + image = gen_2D_image(img_path, image_x, image_y, pixel_type) + ground_truth_path = ground_truth_dir.joinpath("ground_truth.ome.tif") + gen_ground_truth(img_path, ground_truth_path) + + yield inp_dir, out_dir, ground_truth_dir, img_path, ground_truth_path + + shutil.rmtree(data_dir) + +def gen_2D_image( + img_path, + image_x, + image_y, + pixel_type +) : + """Generate a random 2D square image.""" + + if np.issubdtype(pixel_type, np.floating) : + rng = np.random.default_rng() + image = rng.uniform(0.0, 1.0, + size=(image_y, image_x) + ).astype(pixel_type) + else: + image = np.random.randint(0, 255, size=(image_y, image_x)) + + with BioWriter(img_path) as writer: + (y, x) = image.shape + writer.Y = y + writer.X = x + writer.Z = 1 + writer.C = 1 + writer.T = 1 + writer.dtype = image.dtype + writer[:] = image[:] + + return image + + +def gen_ground_truth(img_path : Path, ground_truth_path : Path): + """generate some ground truth from the image data. + Here we generate a simple binary mask. + """ + + with BioReader(img_path) as reader: + with BioWriter(ground_truth_path, metadata=reader.metadata) as writer: + ground_truth = np.asarray(reader[:] != 0) + writer[:] = ground_truth + + return ground_truth \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/tests/test_cli.py b/utils/polus-python-template/{{cookiecutter.container_name}}/tests/test_cli.py new file mode 100644 index 000000000..1b5180971 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/tests/test_cli.py @@ -0,0 +1,96 @@ +"""Testing the Command Line Tool.""" + +import faulthandler +import json +from pathlib import Path +from typer.testing import CliRunner + +from .conftest import FixtureReturnType + +from {{cookiecutter.plugin_package}}.__main__ import app + +faulthandler.enable() + + +def test_cli(generate_test_data : FixtureReturnType) -> None: # noqa + """Test the command line.""" + inp_dir, out_dir, ground_truth_dir, img_path, ground_truth_path = generate_test_data #noqa + + runner = CliRunner() + + result = runner.invoke( + app, + [ + "--inpDir", + inp_dir, + "--outDir", + out_dir, + ], + ) + + assert result.exit_code == 0 + +def test_cli_short(generate_test_data : FixtureReturnType): # noqa + """Test the command line.""" + runner = CliRunner() + + inp_dir, out_dir, _, _, _ = generate_test_data #noqa + + result = runner.invoke( + app, + [ + "-i", + inp_dir, + "-o", + out_dir, + ], + ) + + assert result.exit_code == 0 + +def test_cli_preview(generate_test_data : FixtureReturnType): # noqa + """Test the preview option.""" + runner = CliRunner() + + inp_dir, out_dir, _, _, _ = generate_test_data #noqa + + + result = runner.invoke( + app, + [ + "--inpDir", + inp_dir, + "--outDir", + out_dir, + "--preview", + ], + ) + + assert result.exit_code == 0 + + with Path.open(out_dir / "preview.json") as file: + plugin_json = json.load(file) + + # verify we generate the preview file + assert plugin_json == {} + + +def test_cli_bad_input(generate_test_data : FixtureReturnType): # noqa + """Test bad inputs.""" + runner = CliRunner() + + inp_dir, out_dir, _, _, _ = generate_test_data #noqa + # replace with a bad path + inp_dir = "/does_not_exists" + + result = runner.invoke( + app, + [ + "--inpDir", + inp_dir, + "--outDir", + out_dir, + ], + ) + + assert result.exc_info[0] is SystemExit diff --git a/utils/polus-python-template/{{cookiecutter.container_name}}/tests/test_{{cookiecutter.package_name}}.py b/utils/polus-python-template/{{cookiecutter.container_name}}/tests/test_{{cookiecutter.package_name}}.py new file mode 100644 index 000000000..75e3552e2 --- /dev/null +++ b/utils/polus-python-template/{{cookiecutter.container_name}}/tests/test_{{cookiecutter.package_name}}.py @@ -0,0 +1,22 @@ +"""Tests for {{cookiecutter.package_name}}.""" + +import pytest +from {{cookiecutter.plugin_package}}.{{cookiecutter.package_name}} import ( + {{cookiecutter.package_name}}, +) +from .conftest import FixtureReturnType + + +def test_{{cookiecutter.package_name}}(generate_test_data : FixtureReturnType): + """Test {{cookiecutter.package_name}}.""" + inp_dir, out_dir, ground_truth_dir, img_path, ground_truth_path = generate_test_data + filepattern = ".*" + assert {{cookiecutter.package_name}}(inp_dir, filepattern, out_dir) == None + + +@pytest.mark.skipif("not config.getoption('slow')") +def test_{{cookiecutter.package_name}}(generate_large_test_data : FixtureReturnType): + """Test {{cookiecutter.package_name}}.""" + inp_dir, out_dir, ground_truth_dir, img_path, ground_truth_path = generate_large_test_data + filepattern = ".*" + assert {{cookiecutter.package_name}}(inp_dir, filepattern, out_dir) == None \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.project_slug}}/Dockerfile b/utils/polus-python-template/{{cookiecutter.project_slug}}/Dockerfile deleted file mode 100644 index 21d5eede1..000000000 --- a/utils/polus-python-template/{{cookiecutter.project_slug}}/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ - -FROM labshare/polus-bfio-util:2.1.9 - -# environment variables defined in labshare/polus-bfio-util -# ENV EXEC_DIR="/opt/executables" -# ENV DATA_DIR="/data" -# ENV POLUS_EXT=".ome.tif" -# ENV POLUS_LOG="INFO" # Change to WARNING for fewer logs, and DEBUG for debugging - -# Work directory defined in the base container -# WORKDIR ${EXEC_DIR} - -COPY VERSION ${EXEC_DIR} -COPY src ${EXEC_DIR}/ - -RUN pip3 install -r ${EXEC_DIR}/requirements.txt --no-cache-dir - -# Default command. Additional arguments are provided through the command line -ENTRYPOINT ["python3", "main.py"] \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.project_slug}}/README.md b/utils/polus-python-template/{{cookiecutter.project_slug}}/README.md deleted file mode 100644 index bf104dd15..000000000 --- a/utils/polus-python-template/{{cookiecutter.project_slug}}/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# {{ cookiecutter.project_name }} (v{{ cookiecutter.version }}) - -This WIPP plugin does things, some of which involve math and science. There is -likely a lot of handwaving involved when describing how it works, but handwaving -should be replaced with a good description. However, someone forgot to edit the -README, so handwaving will have to do for now. Contact -[{{ cookiecutter.author }}](mailto:{{ cookiecutter.email }}) for more -information. - -For more information on WIPP, visit the -[official WIPP page](https://isg.nist.gov/deepzoomweb/software/wipp). - -## Building - -To build the Docker image for the conversion plugin, run -`./build-docker.sh`. - -## Install WIPP Plugin - -If WIPP is running, navigate to the plugins page and add a new plugin. Paste the -contents of `plugin.json` into the pop-up window and submit. - -## Options - -This plugin takes {{ cookiecutter._inputs|length }} input arguments and -{{ cookiecutter._outputs|length }} output argument: - -| Name | Description | I/O | Type | -|---------------|-------------------------|--------|--------| -{% for inp,val in cookiecutter._inputs.items() %}| `--{{ inp }}` | {{ val.description }} | Input | {{ val.type }} | -{% endfor %}{% for out,val in cookiecutter._outputs|dictsort %}| `--{{ out }}` | {{ val.description }} | Output | {{ val.type }} | -{% endfor %} diff --git a/utils/polus-python-template/{{cookiecutter.project_slug}}/VERSION b/utils/polus-python-template/{{cookiecutter.project_slug}}/VERSION deleted file mode 100644 index a8faae458..000000000 --- a/utils/polus-python-template/{{cookiecutter.project_slug}}/VERSION +++ /dev/null @@ -1 +0,0 @@ -{{ cookiecutter.version }} \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.project_slug}}/build-docker.sh b/utils/polus-python-template/{{cookiecutter.project_slug}}/build-docker.sh deleted file mode 100755 index 55c344d9b..000000000 --- a/utils/polus-python-template/{{cookiecutter.project_slug}}/build-docker.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -version=$( np.ndarray: - """Awesome function (actually just a template) - - This function should do something, but for now just returns the input. - - Args: - input_data: A numpy array. - - Returns: - np.ndarray: Returns the input image. - """ - - return input_data - -{# This generates the main function definition -#} -{#- Required inputs are arguments -#} -{#- Optional inputs are keyword arguments -#} -def main({#- Required inputs -#} - {% for inp,val in cookiecutter._inputs.items() -%} - {% if val.required -%} - {% if val.type=="boolean" -%} - {{ inp }}: bool, - {% elif val.type=="collection" and cookiecutter.use_bfio -%} - {{ inp }}: Path, - {% else -%} - {{ inp }}: str, - {% endif -%} - {% endif -%} - {% endfor -%} - {#- Required Outputs (all outputs are required, and should be a Path) -#} - {% for inp,val in cookiecutter._outputs.items() -%} - {{ inp }}: Path, - {% endfor -%} - {#- Optional inputs -#} - {% for inp,val in cookiecutter._inputs.items() -%} - {% if not val.required -%} - {% if val.type=="boolean" -%} - {{ inp }}: typing.Optional[bool] = None, - {% elif val.type=="collection" and cookiecutter.use_bfio -%} - {{ inp }}: typing.Options[Path] = None, - {% else -%} - {{ inp }}: typing.Optional[str] = None, - {% endif -%} - {% endif -%} - {% endfor -%}) -> None: - """ Main execution function - - All functions in your code must have docstrings attached to them, and the - docstrings must follow the Google Python Style: - https://www.sphinx-doc.org/en/master/usage/extensions/example_google.html - """ - {# Initialize a filepattern object if filepattern is going to be used -#} - {%- if cookiecutter.use_filepattern == "True" %} - pattern = filePattern if filePattern is not None else '.*' - fp = filepattern.FilePattern( - {%- for inp,val in cookiecutter._inputs.items() if val.type=='collection' -%} - {% if loop.first %}{{ inp }}{% endif %} - {%- endfor -%},pattern) - - for files in fp: - # get the first file - file = files.pop() - - {%- else %} - files = list({% for inp,val in cookiecutter._inputs.items() if val.type=='collection' -%} - {% if loop.first %}{{ inp }}{% endif %} - {%- endfor -%}.iterdir()) - - for file in files: - - {%- endif %} - {#- Use bfio if requested #} - {%- if cookiecutter.use_bfio == "True" %} - {%- filter indent(level2,True) %} - - logger.info(f'Processing image: {file["file"]}') - - # Load the input image - logger.debug(f'Initializing BioReader for {file["file"]}') - with BioReader(file['file']) as br: - - input_extension = ''.join([s for s in file['file'].suffixes[-2:] if len(s) < 5]) - out_name = file['file'].name.replace(input_extension,POLUS_EXT) - out_path = {{ cookiecutter._outputs.keys()|first }}.joinpath(out_name) - - # Initialize the output image - logger.debug(f'Initializing BioReader for {out_path}') - with BioWriter(out_path,metadata=br.metadata) as bw: - - # This is where the magic happens, replace this part with your method - bw[:] = awesome_function(br[:]) - {%- endfilter %} - {%- endif %} - -if __name__=="__main__": - - ''' Argument parsing ''' - logger.info("Parsing arguments...") - parser = argparse.ArgumentParser(prog='main', description='{{ cookiecutter.project_short_description }}') - - # Input arguments - {% for inp,val in cookiecutter._inputs.items() -%} - parser.add_argument('--{{ inp }}', dest='{{ inp }}', type=str, - help='{{ val.description }}', required={{ val.required }}) - {% endfor -%} - - # Output arguments - {%- for out,val in cookiecutter._outputs.items() %} - parser.add_argument('--{{ out }}', dest='{{ out }}', type=str, - help='{{ val.description }}', required=True) - {% endfor %} - # Parse the arguments - args = parser.parse_args() - {% for inp,val in cookiecutter._inputs.items() -%} - {% if val.type=="boolean" -%} - {{ inp }} = args.{{ inp }} == 'true' - logger.info('{{ inp }} = {}'.format({{ inp }})) - {% elif val.type=="collection" -%} - {{ inp }} = Path(args.{{ inp }}) - if ({{ inp }}.joinpath('images').is_dir()): - # switch to images folder if present - {{ inp }} = {{ inp }}.joinpath('images').absolute() - {% else -%} - {{ inp }} = args.{{ inp }} - {% endif -%} - logger.info('{{ inp }} = {}'.format({{ inp }})) - {% endfor %} - {%- for out,val in cookiecutter._outputs.items() -%} - {{ out }} = Path(args.{{ out }}) - logger.info('{{ out }} = {}'.format({{ out }})) - {%- endfor %} - - main( - {%- filter indent(5) %} - {%- for inp,val in cookiecutter._inputs.items() -%} - {{ inp }}={{ inp }}, - {% endfor -%} - {%- for inp,val in cookiecutter._outputs.items() -%} - {{ inp }}={{ inp }}{% if not loop.last %},{% endif %}{% endfor %}{% endfilter -%} - ) \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.project_slug}}/src/requirements.txt b/utils/polus-python-template/{{cookiecutter.project_slug}}/src/requirements.txt deleted file mode 100644 index b1dac1085..000000000 --- a/utils/polus-python-template/{{cookiecutter.project_slug}}/src/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -bfio[all]==2.1.9 -filepattern==1.4.6 \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/__init__.py b/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/__init__.py deleted file mode 100644 index 743a7355d..000000000 --- a/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from unittest import TestSuite -from .version_test import VersionTest -from .plugin_test import PluginTest - -test_cases = (VersionTest, - PluginTest) - -def load_tests(loader, tests, pattern): - suite = TestSuite() - for test_class in test_cases: - tests = loader.loadTestsFromTestCase(test_class) - suite.addTests(tests) - return suite \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/plugin_test.py b/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/plugin_test.py deleted file mode 100644 index baa00c384..000000000 --- a/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/plugin_test.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest - -class PluginTest(unittest.TestCase): - """ Tests to ensure the plugin is operating correctly """ - - def test_sanity(self): - - self.fail('No plugin tests were created.') - -if __name__=="__main__": - - unittest.main() \ No newline at end of file diff --git a/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/version_test.py b/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/version_test.py deleted file mode 100644 index c3bd8f704..000000000 --- a/utils/polus-python-template/{{cookiecutter.project_slug}}/tests/version_test.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest, json -from pathlib import Path -import urllib.request as request - -class VersionTest(unittest.TestCase): - """ Verify VERSION is correct """ - - version_path = Path(__file__).parent.parent.joinpath("VERSION") - json_path = Path(__file__).parent.parent.joinpath("plugin.json") - url = 'https://hub.docker.com/v2/repositories/labshare/{{ cookiecutter.project_slug }}/tags/?page_size=1&page=1&ordering=last_updated' - - def test_plugin_manifest(self): - """ Tests VERSION matches the version in the plugin manifest """ - - # Get the plugin version - with open(self.version_path,'r') as file: - version = file.readline() - - # Load the plugin manifest - with open(self.json_path,'r') as file: - plugin_json = json.load(file) - - self.assertEqual(plugin_json['version'],version) - self.assertTrue(plugin_json['containerId'].endswith(version)) - - def test_docker_hub(self): - """ Tests VERSION matches the latest docker container tag """ - - # Get the plugin version - with open(self.version_path,'r') as file: - version = file.readline() - - response = json.load(request.urlopen(self.url)) - if len(response['results']) == 0: - self.fail('Could not find repository or no containers are in the repository.') - latest_tag = json.load(response)['results'][0]['name'] - - self.assertEqual(latest_tag,version) - -if __name__=="__main__": - - unittest.main() \ No newline at end of file