From f2dc3a3cb48d5da0f4ce84236777a2e24588ebeb Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Mon, 31 Jul 2023 13:30:52 -0400 Subject: [PATCH 01/10] fix: fixed all tests after filepattern PR --- .../apply-flatfield-plugin/pyproject.toml | 38 +++++ .../images/apply_flatfield/__init__.py | 160 ++++++++++++++++++ .../tests/test_plugin.py | 151 +++++++++++++++++ 3 files changed, 349 insertions(+) create mode 100644 transforms/images/apply-flatfield-plugin/pyproject.toml create mode 100644 transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py create mode 100644 transforms/images/apply-flatfield-plugin/tests/test_plugin.py diff --git a/transforms/images/apply-flatfield-plugin/pyproject.toml b/transforms/images/apply-flatfield-plugin/pyproject.toml new file mode 100644 index 000000000..d4e72a227 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/pyproject.toml @@ -0,0 +1,38 @@ +[tool.poetry] +name = "polus-plugins-transforms-images-apply-flatfield" +version = "2.0.0-dev7" +description = "" +authors = [ + "Nick Schaub ", + "Najib Ishaq " +] +readme = "README.md" +packages = [{include = "polus", from = "src"}] + +[tool.poetry.dependencies] +python = "^3.9" +bfio = { version = "2.1.9", extras = ["all"] } +filepattern = [ + { version = "^2.0.0", platform = "linux" }, + { version = "^2.0.0", platform = "win32" }, + # { git = "https://github.com/PolusAI/filepattern", rev = "c07bf543c435cbc4cf264effd5a178868e9eaf19", platform = "darwin" }, + { git = "https://github.com/JesseMckinzie/filepattern-1", rev = "c27cf04ba3a1946b87c0c43d5720ba394c340894", platform = "darwin" }, +] +typer = { version = "^0.7.0", extras = ["all"] } +numpy = "^1.24.3" +tqdm = "^4.65.0" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.1" +pytest-cov = "^4.0.0" +pytest-sugar = "^0.9.6" +pytest-xdist = "^3.2.0" +pytest-benchmark = "^4.0.0" +bump2version = "^1.0.1" +pre-commit = "^3.0.4" +black = "^23.1.0" +ruff = "^0.0.265" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py new file mode 100644 index 000000000..dd697c5a6 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py @@ -0,0 +1,160 @@ +"""Provides the apply_flatfield module.""" + +import concurrent.futures +import logging +import operator +import pathlib +import sys +import typing + +import bfio +import numpy +import tqdm +from filepattern import FilePattern + +from . import utils + +logger = logging.getLogger(__name__) +logger.setLevel(utils.POLUS_LOG) + + +def apply( # noqa: PLR0913 + img_dir: pathlib.Path, + img_pattern: str, + ff_dir: pathlib.Path, + ff_pattern: str, + df_pattern: typing.Optional[str], + out_dir: pathlib.Path, +) -> None: + """Run batch-wise flatfield correction on the image collection.""" + img_fp = FilePattern(str(img_dir), img_pattern) + img_variables = img_fp.get_variables() + + ff_fp = FilePattern(str(ff_dir), ff_pattern) + ff_variables = ff_fp.get_variables() + + # check that ff_variables are a subset of img_variables + if set(ff_variables) - set(img_variables): + msg = ( + f"Flatfield variables are not a subset of image variables: " + f"{ff_variables} - {img_variables}" + ) + raise ValueError(msg) + + if (df_pattern is None) or (not df_pattern): + df_fp = None + else: + df_fp = FilePattern(str(ff_dir), df_pattern) + df_variables = df_fp.get_variables() + if set(df_variables) != set(ff_variables): + msg = ( + f"Flatfield and darkfield variables do not match: " + f"{ff_variables} != {df_variables}" + ) + raise ValueError(msg) + + for group, files in img_fp(group_by=ff_variables): + img_paths = [p for _, [p] in files] + variables = dict(group) + + ff_path: pathlib.Path = ff_fp.get_matching(**variables)[0][1][0] + + df_path = None if df_fp is None else df_fp.get_matching(**variables)[0][1][0] + + _unshade_images(img_paths, out_dir, ff_path, df_path) + + +def _unshade_images( + img_paths: list[pathlib.Path], + out_dir: pathlib.Path, + ff_path: pathlib.Path, + df_path: typing.Optional[pathlib.Path], +) -> None: + """Remove the given flatfield components from all images and save outputs. + + Args: + img_paths: list of paths to images to be processed + out_dir: directory to save the corrected images + ff_path: path to the flatfield image + df_path: path to the darkfield image + """ + with bfio.BioReader(ff_path, max_workers=2) as bf: + ff_image = bf[:, :, :, 0, 0].squeeze() + + if df_path is not None: + with bfio.BioReader(df_path, max_workers=2) as df: + df_image = df[:, :, :, 0, 0].squeeze() + else: + df_image = None + + batch_indices = list(range(0, len(img_paths), 16)) + if batch_indices[-1] != len(img_paths): + batch_indices.append(len(img_paths)) + + for i_start, i_end in tqdm.tqdm( + zip(batch_indices[:-1], batch_indices[1:]), + total=len(batch_indices) - 1, + ): + _unshade_batch( + img_paths[i_start:i_end], + out_dir, + ff_image, + df_image, + ) + + +def _unshade_batch( + batch_paths: list[pathlib.Path], + out_dir: pathlib.Path, + ff_image: numpy.ndarray, + df_image: typing.Optional[numpy.ndarray] = None, +) -> None: + """Apply flatfield correction to a batch of images. + + Args: + batch_paths: list of paths to images to be processed + out_dir: directory to save the corrected images + ff_image: component to be used for flatfield correction + df_image: component to be used for flatfield correction + """ + # Load images + images = [] + with concurrent.futures.ProcessPoolExecutor( + max_workers=utils.MAX_WORKERS, + ) as load_executor: + load_futures = [] + for i, inp_path in enumerate(batch_paths): + load_futures.append(load_executor.submit(utils.load_img, inp_path, i)) + + for lf in tqdm.tqdm( + concurrent.futures.as_completed(load_futures), + total=len(load_futures), + desc="Loading batch", + ): + images.append(lf.result()) + + images = [img for _, img in sorted(images, key=operator.itemgetter(0))] + img_stack = numpy.stack(images, axis=0) + + # Apply flatfield correction + if df_image is not None: + img_stack -= df_image + + img_stack /= ff_image + + # Save outputs + with concurrent.futures.ProcessPoolExecutor( + max_workers=utils.MAX_WORKERS, + ) as save_executor: + save_futures = [] + for inp_path, img in zip(batch_paths, img_stack): + save_futures.append( + save_executor.submit(utils.save_img, inp_path, img, out_dir), + ) + + for sf in tqdm.tqdm( + concurrent.futures.as_completed(save_futures), + total=len(save_futures), + desc="Saving batch", + ): + sf.result() diff --git a/transforms/images/apply-flatfield-plugin/tests/test_plugin.py b/transforms/images/apply-flatfield-plugin/tests/test_plugin.py new file mode 100644 index 000000000..886773807 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/tests/test_plugin.py @@ -0,0 +1,151 @@ +"""Tests for the plugin.""" + +import itertools +import logging +import pathlib +import shutil +import tempfile + +import bfio +import numpy +import pytest +import typer.testing +from polus.plugins.transforms.images.apply_flatfield import apply +from polus.plugins.transforms.images.apply_flatfield.__main__ import app + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def _make_random_image( + path: pathlib.Path, + rng: numpy.random.Generator, + size: int, +) -> None: + with bfio.BioWriter(path) as writer: + writer.X = size + writer.Y = size + writer.dtype = numpy.float32 + + writer[:] = rng.random(size=(size, size), dtype=writer.dtype) + + +FixtureReturnType = tuple[pathlib.Path, str, pathlib.Path, str] + + +def gen_once(num_groups: int, img_size: int) -> FixtureReturnType: + """Generate a set of random images for testing.""" + + img_pattern = "img_x{x}_c{c}.ome.tif" + ff_pattern = "img_x(1-10)_c{c}" + + img_dir = pathlib.Path(tempfile.mkdtemp(suffix="img_dir")) + ff_dir = pathlib.Path(tempfile.mkdtemp(suffix="ff_dir")) + + rng = numpy.random.default_rng(42) + + for i in range(num_groups): + ff_path = ff_dir.joinpath(f"{ff_pattern.format(c=i + 1)}_flatfield.ome.tif") + _make_random_image(ff_path, rng, img_size) + + df_path = ff_dir.joinpath(f"{ff_pattern.format(c=i + 1)}_darkfield.ome.tif") + _make_random_image(df_path, rng, img_size) + + for j in range(10): # 10 images in each group + img_path = img_dir.joinpath(img_pattern.format(x=j + 1, c=i + 1)) + _make_random_image(img_path, rng, img_size) + + image_names = list(sorted(p.name for p in img_dir.iterdir())) + logger.debug(f"Generated {image_names} images in {img_dir}") + + ff_names = list(sorted(p.name for p in ff_dir.iterdir())) + logger.debug(f"Generated {ff_names} flatfield images in {ff_dir}") + + img_pattern = "img_x{x:d+}_c{c:d}.ome.tif" + ff_pattern = "img_x\\(1-10\\)_c{c:d}" + return img_dir, img_pattern, ff_dir, ff_pattern + + +NUM_GROUPS = [2**i for i in range(3)] +IMG_SIZES = [1024 * 2**i for i in range(3)] +PARAMS = list(itertools.product(NUM_GROUPS, IMG_SIZES)) +IDS = [f"{num_groups}_{img_size}" for num_groups, img_size in PARAMS] + + +@pytest.fixture(params=PARAMS, ids=IDS) +def gen_images(request: pytest.FixtureRequest) -> FixtureReturnType: + """Generate a set of random images for testing.""" + num_groups: int + img_size: int + num_groups, img_size = request.param + img_dir, img_pattern, ff_dir, ff_pattern = gen_once(num_groups, img_size) + + yield img_dir, img_pattern, ff_dir, ff_pattern + + # Cleanup + shutil.rmtree(img_dir) + shutil.rmtree(ff_dir) + + +def test_estimate(gen_images: FixtureReturnType) -> None: + """Test the `estimate` function.""" + + img_dir, img_pattern, ff_dir, ff_pattern = gen_images + out_dir = pathlib.Path(tempfile.mkdtemp(suffix="out_dir")) + + apply( + img_dir, + img_pattern, + ff_dir, + f"{ff_pattern}_flatfield.ome.tif", + f"{ff_pattern}_darkfield.ome.tif", + out_dir, + ) + + img_names = [p.name for p in img_dir.iterdir()] + out_names = [p.name for p in out_dir.iterdir()] + + for name in img_names: + assert name in out_names, f"{name} not in {out_names}" + + shutil.rmtree(out_dir) + + +def test_cli() -> None: + """Test the CLI.""" + + img_dir, img_pattern, ff_dir, ff_pattern = gen_once(2, 2_048) + out_dir = pathlib.Path(tempfile.mkdtemp(suffix="out_dir")) + + runner = typer.testing.CliRunner() + + result = runner.invoke( + app, + [ + "--imgDir", + str(img_dir), + "--imgPattern", + img_pattern, + "--ffDir", + str(ff_dir), + "--brightPattern", + f"{ff_pattern}_flatfield.ome.tif", + "--darkPattern", + f"{ff_pattern}_darkfield.ome.tif", + "--outDir", + str(out_dir), + ], + ) + + assert result.exit_code == 0, result.stdout + + img_paths = set(p.name for p in img_dir.iterdir() if p.name.endswith(".ome.tif")) + + out_names = set(p.name for p in out_dir.iterdir() if p.name.endswith(".ome.tif")) + + assert img_paths == out_names, f"{(img_paths)} != {out_names}" + + # Cleanup + shutil.rmtree(img_dir) + shutil.rmtree(ff_dir) + shutil.rmtree(out_dir) From 79a9898a276114c7c0bdc109a8aad8776c264117 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Mon, 31 Jul 2023 14:21:03 -0400 Subject: [PATCH 02/10] build: bump version 2.0.0-dev7 -> 2.0.0-dev8 --- .../apply-flatfield-plugin/.bumpversion.cfg | 29 ++++++ .../images/apply-flatfield-plugin/README.md | 49 ++++++++++ .../images/apply-flatfield-plugin/VERSION | 1 + .../images/apply-flatfield-plugin/plugin.json | 94 +++++++++++++++++++ .../apply-flatfield-plugin/pyproject.toml | 2 +- .../images/apply_flatfield/__init__.py | 2 + 6 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 transforms/images/apply-flatfield-plugin/.bumpversion.cfg create mode 100644 transforms/images/apply-flatfield-plugin/README.md create mode 100644 transforms/images/apply-flatfield-plugin/VERSION create mode 100644 transforms/images/apply-flatfield-plugin/plugin.json diff --git a/transforms/images/apply-flatfield-plugin/.bumpversion.cfg b/transforms/images/apply-flatfield-plugin/.bumpversion.cfg new file mode 100644 index 000000000..a0dda9006 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/.bumpversion.cfg @@ -0,0 +1,29 @@ +[bumpversion] +current_version = 2.0.0-dev8 +commit = False +tag = False +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? +serialize = + {major}.{minor}.{patch}-{release}{dev} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = _ +first_value = dev +values = + dev + _ + +[bumpversion:part:dev] + +[bumpversion:file:pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" + +[bumpversion:file:plugin.json] + +[bumpversion:file:VERSION] + +[bumpversion:file:README.md] + +[bumpversion:file:src/polus/plugins/transforms/images/apply_flatfield/__init__.py] diff --git a/transforms/images/apply-flatfield-plugin/README.md b/transforms/images/apply-flatfield-plugin/README.md new file mode 100644 index 000000000..8ac8366c9 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/README.md @@ -0,0 +1,49 @@ +# Apply Flatfield Plugin (v2.0.0-dev8) + +This WIPP plugin applies a flatfield operation on every image in a collection. +The algorithm used to apply the flatfield is as follows: + +![Corrected = \frac{Original - Darkfield}{Brightfield} - Photobleach + Offset](https://render.githubusercontent.com/render/math?math=Corrected%20%3D%20%5Cfrac%7BOriginal%20-%20Darkfield%7D%7BBrightfield%7D%20-%20Photobleach%20%2B%20Offset) + +A brief description of the variables: +1. ![Corrected](https://render.githubusercontent.com/render/math?math=Corrected) is the flatfield corrected image. +2. ![Darkfield](https://render.githubusercontent.com/render/math?math=Darkfield) is the darkfield image (sometimes referred to as offset, dark current, or dark noise). This is an image collected by the camera when the shutter is closed. +3. ![Brightfield](https://render.githubusercontent.com/render/math?math=Brightfield) is the normalized brightfield image. This is an image collected when the shutter is open and illumination is on. The image should contain single precision floating point values, where ![mean(Brightfield)=1](https://render.githubusercontent.com/render/math?math=mean(Brightfield)%3D1). +4. ![Photobleach](https://render.githubusercontent.com/render/math?math=Photobleach) is a scalar indicating how much the image has been photobleached. This is a per-image scalar offset. +5. ![Offset](https://render.githubusercontent.com/render/math?math=Offset) is a scalar applied to all images in the collection. If ![Photobleach](https://render.githubusercontent.com/render/math?math=Photobleach) is specified, then this plugin uses ![Offset=mean(Photobleach)](https://render.githubusercontent.com/render/math?math=Offset%3Dmean(Photobleach)). + +For more information on flatfielding, see the paper by [Young](https://currentprotocols.onlinelibrary.wiley.com/doi/full/10.1002/0471142956.cy0211s14). +This plugin specifically uses the formulation from [Peng et al](https://www.nature.com/articles/ncomms14836). + +For more information on WIPP, visit the +[official WIPP page](https://isg.nist.gov/deepzoomweb/software/wipp). + +## To Do + +Implement additional formulations of flatfield correction. Specifically, the formula specified by Young: + +![Corrected = \frac{Original - Darkfield}{Brightfield - Darkfield} ](https://render.githubusercontent.com/render/math?math=Corrected%20%3D%20%5Cfrac%7BOriginal%20-%20Darkfield%7D%7BBrightfield%20-%20Darkfield%7D%20) + +Additional formulations may also include reference image free algorithms for flatfield correction, such as the [rolling ball algorithm](https://www.computer.org/csdl/magazine/co/1983/01/01654163/13rRUwwJWBB). + +## Building + +To build the Docker image for the conversion plugin, run `./build-docker.sh`. + +## Install WIPP Plugin + +If WIPP is running, navigate to the plugins page and add a new plugin. +Paste the contents of `plugin.json` into the pop-up window and submit. + +## Options + +Command line options: + +| Name | Description | I/O | Type | +|------------------|-----------------------------------------------------------------------|--------|------------| +| `--darkPattern` | Filename pattern used to match darkfield files to image files | Input | string | +| `--ffDir` | Image collection containing flatfield and/or darkfield images | Input | collection | +| `--flatPattern` | Filename pattern used to match flatfield files to image files | Input | string | +| `--imgDir` | Input image collection to be processed by this plugin | Input | collection | +| `--imgPattern` | Filename pattern used to separate data and match with flatfield files | Input | string | +| `--outDir` | Output collection | Output | collection | diff --git a/transforms/images/apply-flatfield-plugin/VERSION b/transforms/images/apply-flatfield-plugin/VERSION new file mode 100644 index 000000000..e7940151c --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/VERSION @@ -0,0 +1 @@ +2.0.0-dev8 diff --git a/transforms/images/apply-flatfield-plugin/plugin.json b/transforms/images/apply-flatfield-plugin/plugin.json new file mode 100644 index 000000000..871525457 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/plugin.json @@ -0,0 +1,94 @@ +{ + "name": "Apply Flatfield", + "version": "2.0.0-dev8", + "title": "Apply Flatfield", + "description": "Apply a flatfield algorithm to a collection of images.", + "author": "Nick Schaub (Nick.Schaub@nih.gov), Najib Ishaq (najib.ishaq@nih.gov)", + "institution": "National Center for Advancing Translational Sciences, National Institutes of Health", + "repository": "https://github.com/labshare/polus-plugins", + "website": "https://ncats.nih.gov/preclinical/core/informatics", + "citation": "", + "containerId": "polusai/apply-flatfield-plugin:2.0.0-dev8", + "baseCommand": [ + "python3", + "-m", + "polus.plugins.transforms.images.apply_flatfield" + ], + "inputs": [ + { + "name": "darkPattern", + "type": "string", + "description": "Filename pattern used to match darkfield files to image files", + "required": false + }, + { + "name": "ffDir", + "type": "collection", + "description": "Image collection containing flatfield and/or darkfield images", + "required": true + }, + { + "name": "brightPattern", + "type": "string", + "description": "Filename pattern used to match brightfield files to image files", + "required": true + }, + { + "name": "imgDir", + "type": "collection", + "description": "Input image collection to be processed by this plugin", + "required": true + }, + { + "name": "imgPattern", + "type": "string", + "description": "Filename pattern used to separate data and match with flatfied files", + "required": true + }, + { + "name": "photoPattern", + "type": "string", + "description": "Filename pattern used to match photobleach files to image files", + "required": true + } + ], + "outputs": [ + { + "name": "outDir", + "type": "collection", + "description": "Output collection" + } + ], + "ui": [ + { + "key": "inputs.imgDir", + "title": "Images to correct", + "description": "Input image collection to be processed by this plugin" + }, + { + "key": "inputs.imgPattern", + "title": "Image pattern", + "description": "Filename pattern used to separate data and match with flatfied files" + }, + { + "key": "inputs.ffDir", + "title": "Background images (flatfield/darkfield)", + "description": "Image collection containing flatfield and/or darkfield images" + }, + { + "key": "inputs.brightPattern", + "title": "Brightfield file pattern", + "description": "Filename pattern used to match brightfield files to image files" + }, + { + "key": "inputs.darkPattern", + "title": "Darkfield file pattern", + "description": "Filename pattern used to match darkfield files to image files" + }, + { + "key": "inputs.photoPattern", + "title": "Photobleach file pattern", + "description": "Filename pattern used to match photobleach files to image files" + } + ] +} diff --git a/transforms/images/apply-flatfield-plugin/pyproject.toml b/transforms/images/apply-flatfield-plugin/pyproject.toml index d4e72a227..69408fbb2 100644 --- a/transforms/images/apply-flatfield-plugin/pyproject.toml +++ b/transforms/images/apply-flatfield-plugin/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-plugins-transforms-images-apply-flatfield" -version = "2.0.0-dev7" +version = "2.0.0-dev8" description = "" authors = [ "Nick Schaub ", diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py index dd697c5a6..b23e1fea8 100644 --- a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py @@ -14,6 +14,8 @@ from . import utils +__version__ = "2.0.0-dev8" + logger = logging.getLogger(__name__) logger.setLevel(utils.POLUS_LOG) From 102bdcbbc4c43fcc955581a684e9a0afa8461bd0 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Mon, 31 Jul 2023 14:36:33 -0400 Subject: [PATCH 03/10] fix: run-plugin script --- .../images/apply-flatfield-plugin/Dockerfile | 20 ++ .../images/apply-flatfield-plugin/README.md | 18 +- .../apply-flatfield-plugin/pyproject.toml | 25 +-- .../apply-flatfield-plugin/run-plugin.sh | 29 +++ .../images/apply_flatfield/__init__.py | 158 +-------------- .../images/apply_flatfield/__main__.py | 108 +++++++++++ .../images/apply_flatfield/apply_flatfield.py | 183 ++++++++++++++++++ .../images/apply_flatfield/utils.py | 51 +++++ .../tests/test_plugin.py | 20 +- 9 files changed, 425 insertions(+), 187 deletions(-) create mode 100644 transforms/images/apply-flatfield-plugin/Dockerfile create mode 100755 transforms/images/apply-flatfield-plugin/run-plugin.sh create mode 100644 transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__main__.py create mode 100644 transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py create mode 100644 transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/utils.py diff --git a/transforms/images/apply-flatfield-plugin/Dockerfile b/transforms/images/apply-flatfield-plugin/Dockerfile new file mode 100644 index 000000000..97ef35df9 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/Dockerfile @@ -0,0 +1,20 @@ +FROM polusai/bfio:2.3.3 + +# environment variables defined in polusai/bfio +ENV EXEC_DIR="/opt/executables" +ENV POLUS_IMG_EXT=".ome.tif" +ENV POLUS_TAB_EXT=".csv" +ENV POLUS_LOG="INFO" + +# Work directory defined in the base container +WORKDIR ${EXEC_DIR} + +COPY pyproject.toml ${EXEC_DIR} +COPY VERSION ${EXEC_DIR} +COPY README.md ${EXEC_DIR} +COPY src ${EXEC_DIR}/src + +RUN pip3 install ${EXEC_DIR} --no-cache-dir + +ENTRYPOINT ["python3", "-m", "polus.plugins.transforms.images.apply_flatfield"] +CMD ["--help"] diff --git a/transforms/images/apply-flatfield-plugin/README.md b/transforms/images/apply-flatfield-plugin/README.md index 8ac8366c9..68ae4344c 100644 --- a/transforms/images/apply-flatfield-plugin/README.md +++ b/transforms/images/apply-flatfield-plugin/README.md @@ -18,13 +18,20 @@ This plugin specifically uses the formulation from [Peng et al](https://www.natu For more information on WIPP, visit the [official WIPP page](https://isg.nist.gov/deepzoomweb/software/wipp). -## To Do +## TODO + +### Additional Flatfield Formulations Implement additional formulations of flatfield correction. Specifically, the formula specified by Young: ![Corrected = \frac{Original - Darkfield}{Brightfield - Darkfield} ](https://render.githubusercontent.com/render/math?math=Corrected%20%3D%20%5Cfrac%7BOriginal%20-%20Darkfield%7D%7BBrightfield%20-%20Darkfield%7D%20) -Additional formulations may also include reference image free algorithms for flatfield correction, such as the [rolling ball algorithm](https://www.computer.org/csdl/magazine/co/1983/01/01654163/13rRUwwJWBB). +Additional formulations may also include reference image free algorithms for flatfield correction, such as the [rolling ball algorithm](https://www.computer.org/csdl/magazine/co/1983/01/01654163/13rRUwwJWBB). + +### Photobleach Correction + +Since the `basicpy` package and the `basic-flatfield-estimation` tool do not yet support photobleach estimation, this plugin does not yet support photobleach correction. +Once they add support for photobleach estimation, this plugin should be updated to support it. ## Building @@ -41,9 +48,10 @@ Command line options: | Name | Description | I/O | Type | |------------------|-----------------------------------------------------------------------|--------|------------| -| `--darkPattern` | Filename pattern used to match darkfield files to image files | Input | string | -| `--ffDir` | Image collection containing flatfield and/or darkfield images | Input | collection | -| `--flatPattern` | Filename pattern used to match flatfield files to image files | Input | string | | `--imgDir` | Input image collection to be processed by this plugin | Input | collection | | `--imgPattern` | Filename pattern used to separate data and match with flatfield files | Input | string | +| `--ffDir` | Image collection containing flatfield and/or darkfield images | Input | collection | +| `--ffPattern` | Filename pattern used to match flatfield files to image files | Input | string | +| `--dfPattern` | Filename pattern used to match darkfield files to image files | Input | string | | `--outDir` | Output collection | Output | collection | +| `--preview` | preview tha output images' names without actually running computation | Input | boolean | diff --git a/transforms/images/apply-flatfield-plugin/pyproject.toml b/transforms/images/apply-flatfield-plugin/pyproject.toml index 69408fbb2..df1c15db4 100644 --- a/transforms/images/apply-flatfield-plugin/pyproject.toml +++ b/transforms/images/apply-flatfield-plugin/pyproject.toml @@ -10,29 +10,24 @@ readme = "README.md" packages = [{include = "polus", from = "src"}] [tool.poetry.dependencies] -python = "^3.9" -bfio = { version = "2.1.9", extras = ["all"] } -filepattern = [ - { version = "^2.0.0", platform = "linux" }, - { version = "^2.0.0", platform = "win32" }, - # { git = "https://github.com/PolusAI/filepattern", rev = "c07bf543c435cbc4cf264effd5a178868e9eaf19", platform = "darwin" }, - { git = "https://github.com/JesseMckinzie/filepattern-1", rev = "c27cf04ba3a1946b87c0c43d5720ba394c340894", platform = "darwin" }, -] +python = ">=3.9,<3.12" +bfio = { version = "^2.3.3", extras = ["all"] } +filepattern = "^2.0.4" typer = { version = "^0.7.0", extras = ["all"] } numpy = "^1.24.3" tqdm = "^4.65.0" [tool.poetry.group.dev.dependencies] -pytest = "^7.2.1" -pytest-cov = "^4.0.0" -pytest-sugar = "^0.9.6" -pytest-xdist = "^3.2.0" -pytest-benchmark = "^4.0.0" bump2version = "^1.0.1" pre-commit = "^3.0.4" -black = "^23.1.0" -ruff = "^0.0.265" +pytest = "^7.2.1" +pytest-sugar = "^0.9.6" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +pythonpath = [ + "." +] diff --git a/transforms/images/apply-flatfield-plugin/run-plugin.sh b/transforms/images/apply-flatfield-plugin/run-plugin.sh new file mode 100755 index 000000000..8509f25ce --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/run-plugin.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +version=$( None: - """Run batch-wise flatfield correction on the image collection.""" - img_fp = FilePattern(str(img_dir), img_pattern) - img_variables = img_fp.get_variables() - - ff_fp = FilePattern(str(ff_dir), ff_pattern) - ff_variables = ff_fp.get_variables() - - # check that ff_variables are a subset of img_variables - if set(ff_variables) - set(img_variables): - msg = ( - f"Flatfield variables are not a subset of image variables: " - f"{ff_variables} - {img_variables}" - ) - raise ValueError(msg) - - if (df_pattern is None) or (not df_pattern): - df_fp = None - else: - df_fp = FilePattern(str(ff_dir), df_pattern) - df_variables = df_fp.get_variables() - if set(df_variables) != set(ff_variables): - msg = ( - f"Flatfield and darkfield variables do not match: " - f"{ff_variables} != {df_variables}" - ) - raise ValueError(msg) - - for group, files in img_fp(group_by=ff_variables): - img_paths = [p for _, [p] in files] - variables = dict(group) - - ff_path: pathlib.Path = ff_fp.get_matching(**variables)[0][1][0] - - df_path = None if df_fp is None else df_fp.get_matching(**variables)[0][1][0] - - _unshade_images(img_paths, out_dir, ff_path, df_path) - - -def _unshade_images( - img_paths: list[pathlib.Path], - out_dir: pathlib.Path, - ff_path: pathlib.Path, - df_path: typing.Optional[pathlib.Path], -) -> None: - """Remove the given flatfield components from all images and save outputs. - - Args: - img_paths: list of paths to images to be processed - out_dir: directory to save the corrected images - ff_path: path to the flatfield image - df_path: path to the darkfield image - """ - with bfio.BioReader(ff_path, max_workers=2) as bf: - ff_image = bf[:, :, :, 0, 0].squeeze() - - if df_path is not None: - with bfio.BioReader(df_path, max_workers=2) as df: - df_image = df[:, :, :, 0, 0].squeeze() - else: - df_image = None - - batch_indices = list(range(0, len(img_paths), 16)) - if batch_indices[-1] != len(img_paths): - batch_indices.append(len(img_paths)) - - for i_start, i_end in tqdm.tqdm( - zip(batch_indices[:-1], batch_indices[1:]), - total=len(batch_indices) - 1, - ): - _unshade_batch( - img_paths[i_start:i_end], - out_dir, - ff_image, - df_image, - ) - - -def _unshade_batch( - batch_paths: list[pathlib.Path], - out_dir: pathlib.Path, - ff_image: numpy.ndarray, - df_image: typing.Optional[numpy.ndarray] = None, -) -> None: - """Apply flatfield correction to a batch of images. - - Args: - batch_paths: list of paths to images to be processed - out_dir: directory to save the corrected images - ff_image: component to be used for flatfield correction - df_image: component to be used for flatfield correction - """ - # Load images - images = [] - with concurrent.futures.ProcessPoolExecutor( - max_workers=utils.MAX_WORKERS, - ) as load_executor: - load_futures = [] - for i, inp_path in enumerate(batch_paths): - load_futures.append(load_executor.submit(utils.load_img, inp_path, i)) - - for lf in tqdm.tqdm( - concurrent.futures.as_completed(load_futures), - total=len(load_futures), - desc="Loading batch", - ): - images.append(lf.result()) - - images = [img for _, img in sorted(images, key=operator.itemgetter(0))] - img_stack = numpy.stack(images, axis=0) - - # Apply flatfield correction - if df_image is not None: - img_stack -= df_image - - img_stack /= ff_image - - # Save outputs - with concurrent.futures.ProcessPoolExecutor( - max_workers=utils.MAX_WORKERS, - ) as save_executor: - save_futures = [] - for inp_path, img in zip(batch_paths, img_stack): - save_futures.append( - save_executor.submit(utils.save_img, inp_path, img, out_dir), - ) - - for sf in tqdm.tqdm( - concurrent.futures.as_completed(save_futures), - total=len(save_futures), - desc="Saving batch", - ): - sf.result() diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__main__.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__main__.py new file mode 100644 index 000000000..7dac49e72 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__main__.py @@ -0,0 +1,108 @@ +"""Provides the CLI for the Apply Flatfield plugin.""" + +import json +import logging +import pathlib +import typing + +import typer +from polus.plugins.transforms.images.apply_flatfield import apply +from polus.plugins.transforms.images.apply_flatfield import utils + +# Initialize the logger +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +logger = logging.getLogger("polus.plugins.transforms.images.apply_flatfield") +logger.setLevel(utils.POLUS_LOG) + +app = typer.Typer() + + +@app.command() +def main( # noqa: PLR0913 + img_dir: pathlib.Path = typer.Option( + ..., + "--imgDir", + help="Path to input images.", + exists=True, + readable=True, + resolve_path=True, + file_okay=False, + ), + img_pattern: str = typer.Option( + ..., + "--imgPattern", + help="Filename pattern used to select images from imgDir.", + ), + ff_dir: pathlib.Path = typer.Option( + ..., + "--ffDir", + help="Path to flatfield (and optionally darkfield) images.", + exists=True, + readable=True, + resolve_path=True, + file_okay=False, + ), + ff_pattern: str = typer.Option( + ..., + "--ffPattern", + help="Filename pattern used to select flatfield components from ffDir.", + ), + df_pattern: typing.Optional[str] = typer.Option( + None, + "--dfPattern", + help="Filename pattern used to select darkfield components from ffDir.", + ), + out_dir: pathlib.Path = typer.Option( + ..., + "--outDir", + help="Path to output directory.", + exists=True, + writable=True, + resolve_path=True, + file_okay=False, + ), + preview: bool = typer.Option( + False, + "--preview", + help="Preview the output without saving.", + ), +) -> None: + """CLI for the Apply Flatfield plugin. + + The variables used in ffPattern and dfPattern must be a subset of those used + in imgPattern. + + If dfPattern is not specified, then darkfield correction will not be + applied. + """ + logger.info("Starting Apply Flatfield plugin ...") + + logger.info(f"imgDir = {img_dir}") + logger.info(f"imgPattern = {img_pattern}") + logger.info(f"ffDir = {ff_dir}") + logger.info(f"ffPattern = {ff_pattern}") + logger.info(f"dfPattern = {df_pattern}") + logger.info(f"outDir = {out_dir}") + logger.info(f"preview = {preview}") + + out_files = apply( + img_dir=img_dir, + img_pattern=img_pattern, + ff_dir=ff_dir, + ff_pattern=ff_pattern, + df_pattern=df_pattern, + out_dir=out_dir, + preview=preview, + ) + + if preview: + with out_dir.joinpath("preview.json").open("w") as writer: + out_dict = {"files": [p.name for p in out_files]} + json.dump(out_dict, writer, indent=2) + + +if __name__ == "__main__": + app() diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py new file mode 100644 index 000000000..32ddca010 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py @@ -0,0 +1,183 @@ +"""Provides the function to apply flatfield.""" + +import concurrent.futures +import logging +import operator +import pathlib +import typing + +import bfio +import numpy +import tqdm +from filepattern import FilePattern + +from . import utils + +logger = logging.getLogger(__name__) +logger.setLevel(utils.POLUS_LOG) + + +def apply( # noqa: PLR0913 + *, + img_dir: pathlib.Path, + img_pattern: str, + ff_dir: pathlib.Path, + ff_pattern: str, + df_pattern: typing.Optional[str], + out_dir: pathlib.Path, + preview: bool = False, +) -> list[pathlib.Path]: + """Run batch-wise flatfield correction on the image collection. + + Args: + img_dir: path to the directory containing the images to be processed. + img_pattern: filename pattern used to select images from img_dir. + ff_dir: path to the directory containing the flatfield images. + ff_pattern: filename pattern used to select flatfield components from + ff_dir. + df_pattern: filename pattern used to select darkfield components from + ff_dir. + out_dir: path to the directory where the corrected images will be + saved. + preview: if True, return the paths to the images that would be saved + without actually performing any other computation. + """ + img_fp = FilePattern(str(img_dir), img_pattern) + img_variables = img_fp.get_variables() + + ff_fp = FilePattern(str(ff_dir), ff_pattern) + ff_variables = ff_fp.get_variables() + + # check that ff_variables are a subset of img_variables + if set(ff_variables) - set(img_variables): + msg = ( + f"Flatfield variables are not a subset of image variables: " + f"{ff_variables} - {img_variables}" + ) + logger.error(msg) + raise ValueError(msg) + + if (df_pattern is None) or (not df_pattern): + df_fp = None + else: + df_fp = FilePattern(str(ff_dir), df_pattern) + df_variables = df_fp.get_variables() + if set(df_variables) != set(ff_variables): + msg = ( + f"Flatfield and darkfield variables do not match: " + f"{ff_variables} != {df_variables}" + ) + logger.error(msg) + raise ValueError(msg) + + out_files = [] + for group, files in img_fp(group_by=ff_variables): + img_paths = [p for _, [p] in files] + variables = dict(group) + + ff_path: pathlib.Path = ff_fp.get_matching(**variables)[0][1][0] + + df_path = None if df_fp is None else df_fp.get_matching(**variables)[0][1][0] + + if preview: + out_files.extend(img_paths) + else: + _unshade_images(img_paths, out_dir, ff_path, df_path) + + return out_files + + +def _unshade_images( + img_paths: list[pathlib.Path], + out_dir: pathlib.Path, + ff_path: pathlib.Path, + df_path: typing.Optional[pathlib.Path], +) -> None: + """Remove the given flatfield components from all images and save outputs. + + Args: + img_paths: list of paths to images to be processed + out_dir: directory to save the corrected images + ff_path: path to the flatfield image + df_path: path to the darkfield image + """ + with bfio.BioReader(ff_path, max_workers=2) as bf: + ff_image = bf[:, :, :, 0, 0].squeeze() + + if df_path is not None: + with bfio.BioReader(df_path, max_workers=2) as df: + df_image = df[:, :, :, 0, 0].squeeze() + else: + df_image = None + + batch_indices = list(range(0, len(img_paths), 16)) + if batch_indices[-1] != len(img_paths): + batch_indices.append(len(img_paths)) + + for i_start, i_end in tqdm.tqdm( + zip(batch_indices[:-1], batch_indices[1:]), + total=len(batch_indices) - 1, + ): + _unshade_batch( + img_paths[i_start:i_end], + out_dir, + ff_image, + df_image, + ) + + +def _unshade_batch( + batch_paths: list[pathlib.Path], + out_dir: pathlib.Path, + ff_image: numpy.ndarray, + df_image: typing.Optional[numpy.ndarray] = None, +) -> None: + """Apply flatfield correction to a batch of images. + + Args: + batch_paths: list of paths to images to be processed + out_dir: directory to save the corrected images + ff_image: component to be used for flatfield correction + df_image: component to be used for flatfield correction + """ + # Load images + images = [] + with concurrent.futures.ProcessPoolExecutor( + max_workers=utils.MAX_WORKERS, + ) as load_executor: + load_futures = [] + for i, inp_path in enumerate(batch_paths): + load_futures.append(load_executor.submit(utils.load_img, inp_path, i)) + + for lf in tqdm.tqdm( + concurrent.futures.as_completed(load_futures), + total=len(load_futures), + desc="Loading batch", + ): + images.append(lf.result()) + + images = [img for _, img in sorted(images, key=operator.itemgetter(0))] + img_stack = numpy.stack(images, axis=0) + + # Apply flatfield correction + if df_image is not None: + img_stack -= df_image + + img_stack /= ff_image + 1e-8 + + # Save outputs + with concurrent.futures.ProcessPoolExecutor( + max_workers=utils.MAX_WORKERS, + ) as save_executor: + save_futures = [] + for inp_path, img in zip(batch_paths, img_stack): + save_futures.append( + save_executor.submit(utils.save_img, inp_path, img, out_dir), + ) + + for sf in tqdm.tqdm( + concurrent.futures.as_completed(save_futures), + total=len(save_futures), + desc="Saving batch", + ): + sf.result() diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/utils.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/utils.py new file mode 100644 index 000000000..246c9372f --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/utils.py @@ -0,0 +1,51 @@ +"""Utilities for the apply flatfield plugin.""" + +import logging +import multiprocessing +import os +import pathlib + +import bfio +import numpy + +POLUS_LOG = getattr(logging, os.environ.get("POLUS_LOG", "INFO")) +POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") +MAX_WORKERS = max(1, multiprocessing.cpu_count() // 2) + + +def load_img(path: pathlib.Path, i: int) -> tuple[int, numpy.ndarray]: + """Load image from path. + + This method is intended to be used in a thread. The index is used to + identify the image after it has been loaded so that it images can be sorted + in the correct order. + + Args: + path: path to image + i: index of image + """ + with bfio.BioReader(path, MAX_WORKERS) as reader: + image = reader[:, :, :, 0, 0].squeeze() + return i, image + + +def save_img( + inp_path: pathlib.Path, + image: numpy.ndarray, + out_dir: pathlib.Path, +) -> None: + """Save image to disk. + + Args: + inp_path: path to input image + image: image to be saved + out_dir: directory to save image + """ + out_path = out_dir / inp_path.name + with bfio.BioReader(inp_path, MAX_WORKERS) as reader, bfio.BioWriter( + out_path, + MAX_WORKERS, + metadata=reader.metadata, + ) as writer: + writer.dtype = image.dtype + writer[:] = image diff --git a/transforms/images/apply-flatfield-plugin/tests/test_plugin.py b/transforms/images/apply-flatfield-plugin/tests/test_plugin.py index 886773807..8bf296a08 100644 --- a/transforms/images/apply-flatfield-plugin/tests/test_plugin.py +++ b/transforms/images/apply-flatfield-plugin/tests/test_plugin.py @@ -66,8 +66,8 @@ def gen_once(num_groups: int, img_size: int) -> FixtureReturnType: return img_dir, img_pattern, ff_dir, ff_pattern -NUM_GROUPS = [2**i for i in range(3)] -IMG_SIZES = [1024 * 2**i for i in range(3)] +NUM_GROUPS = [1, 4] +IMG_SIZES = [1024, 4096] PARAMS = list(itertools.product(NUM_GROUPS, IMG_SIZES)) IDS = [f"{num_groups}_{img_size}" for num_groups, img_size in PARAMS] @@ -94,12 +94,12 @@ def test_estimate(gen_images: FixtureReturnType) -> None: out_dir = pathlib.Path(tempfile.mkdtemp(suffix="out_dir")) apply( - img_dir, - img_pattern, - ff_dir, - f"{ff_pattern}_flatfield.ome.tif", - f"{ff_pattern}_darkfield.ome.tif", - out_dir, + img_dir=img_dir, + img_pattern=img_pattern, + ff_dir=ff_dir, + ff_pattern=f"{ff_pattern}_flatfield.ome.tif", + df_pattern=f"{ff_pattern}_darkfield.ome.tif", + out_dir=out_dir, ) img_names = [p.name for p in img_dir.iterdir()] @@ -128,9 +128,9 @@ def test_cli() -> None: img_pattern, "--ffDir", str(ff_dir), - "--brightPattern", + "--ffPattern", f"{ff_pattern}_flatfield.ome.tif", - "--darkPattern", + "--dfPattern", f"{ff_pattern}_darkfield.ome.tif", "--outDir", str(out_dir), From 0f8cbfcd6eadf7fdeea294eb35592421d02f8baa Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 21 Dec 2023 13:43:08 -0500 Subject: [PATCH 04/10] added script to build docker image --- transforms/images/apply-flatfield-plugin/build-docker.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 transforms/images/apply-flatfield-plugin/build-docker.sh diff --git a/transforms/images/apply-flatfield-plugin/build-docker.sh b/transforms/images/apply-flatfield-plugin/build-docker.sh new file mode 100644 index 000000000..df2cdd207 --- /dev/null +++ b/transforms/images/apply-flatfield-plugin/build-docker.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +version=$( Date: Thu, 21 Dec 2023 13:43:34 -0500 Subject: [PATCH 05/10] bumped dev version --- transforms/images/apply-flatfield-plugin/.bumpversion.cfg | 2 +- transforms/images/apply-flatfield-plugin/README.md | 2 +- transforms/images/apply-flatfield-plugin/VERSION | 2 +- transforms/images/apply-flatfield-plugin/plugin.json | 4 ++-- transforms/images/apply-flatfield-plugin/pyproject.toml | 2 +- .../plugins/transforms/images/apply_flatfield/__init__.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/transforms/images/apply-flatfield-plugin/.bumpversion.cfg b/transforms/images/apply-flatfield-plugin/.bumpversion.cfg index a0dda9006..cc57c2235 100644 --- a/transforms/images/apply-flatfield-plugin/.bumpversion.cfg +++ b/transforms/images/apply-flatfield-plugin/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.0.0-dev8 +current_version = 2.0.0-dev9 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/transforms/images/apply-flatfield-plugin/README.md b/transforms/images/apply-flatfield-plugin/README.md index 68ae4344c..e67a65258 100644 --- a/transforms/images/apply-flatfield-plugin/README.md +++ b/transforms/images/apply-flatfield-plugin/README.md @@ -1,4 +1,4 @@ -# Apply Flatfield Plugin (v2.0.0-dev8) +# Apply Flatfield Plugin (v2.0.0-dev9) This WIPP plugin applies a flatfield operation on every image in a collection. The algorithm used to apply the flatfield is as follows: diff --git a/transforms/images/apply-flatfield-plugin/VERSION b/transforms/images/apply-flatfield-plugin/VERSION index e7940151c..b2484da91 100644 --- a/transforms/images/apply-flatfield-plugin/VERSION +++ b/transforms/images/apply-flatfield-plugin/VERSION @@ -1 +1 @@ -2.0.0-dev8 +2.0.0-dev9 diff --git a/transforms/images/apply-flatfield-plugin/plugin.json b/transforms/images/apply-flatfield-plugin/plugin.json index 871525457..7605fd52a 100644 --- a/transforms/images/apply-flatfield-plugin/plugin.json +++ b/transforms/images/apply-flatfield-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "Apply Flatfield", - "version": "2.0.0-dev8", + "version": "2.0.0-dev9", "title": "Apply Flatfield", "description": "Apply a flatfield algorithm to a collection of images.", "author": "Nick Schaub (Nick.Schaub@nih.gov), Najib Ishaq (najib.ishaq@nih.gov)", @@ -8,7 +8,7 @@ "repository": "https://github.com/labshare/polus-plugins", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/apply-flatfield-plugin:2.0.0-dev8", + "containerId": "polusai/apply-flatfield-plugin:2.0.0-dev9", "baseCommand": [ "python3", "-m", diff --git a/transforms/images/apply-flatfield-plugin/pyproject.toml b/transforms/images/apply-flatfield-plugin/pyproject.toml index df1c15db4..a8f1b3223 100644 --- a/transforms/images/apply-flatfield-plugin/pyproject.toml +++ b/transforms/images/apply-flatfield-plugin/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-plugins-transforms-images-apply-flatfield" -version = "2.0.0-dev8" +version = "2.0.0-dev9" description = "" authors = [ "Nick Schaub ", diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py index 4b9deea02..7d298350f 100644 --- a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/__init__.py @@ -3,4 +3,4 @@ from . import utils from .apply_flatfield import apply -__version__ = "2.0.0-dev8" +__version__ = "2.0.0-dev9" From 7474075eb1fec5332e6ad1d40aedf9091dce4993 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 21 Dec 2023 13:46:37 -0500 Subject: [PATCH 06/10] fixed plugin manifest --- .../images/apply-flatfield-plugin/README.md | 2 +- .../images/apply-flatfield-plugin/plugin.json | 50 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/transforms/images/apply-flatfield-plugin/README.md b/transforms/images/apply-flatfield-plugin/README.md index e67a65258..37b07eaf0 100644 --- a/transforms/images/apply-flatfield-plugin/README.md +++ b/transforms/images/apply-flatfield-plugin/README.md @@ -54,4 +54,4 @@ Command line options: | `--ffPattern` | Filename pattern used to match flatfield files to image files | Input | string | | `--dfPattern` | Filename pattern used to match darkfield files to image files | Input | string | | `--outDir` | Output collection | Output | collection | -| `--preview` | preview tha output images' names without actually running computation | Input | boolean | +| `--preview` | Preview the output images' names without actually running computation | Input | boolean | diff --git a/transforms/images/apply-flatfield-plugin/plugin.json b/transforms/images/apply-flatfield-plugin/plugin.json index 7605fd52a..ea69873e7 100644 --- a/transforms/images/apply-flatfield-plugin/plugin.json +++ b/transforms/images/apply-flatfield-plugin/plugin.json @@ -16,40 +16,40 @@ ], "inputs": [ { - "name": "darkPattern", - "type": "string", - "description": "Filename pattern used to match darkfield files to image files", - "required": false - }, - { - "name": "ffDir", + "name": "imgDir", "type": "collection", - "description": "Image collection containing flatfield and/or darkfield images", + "description": "Input image collection to be processed by this plugin", "required": true }, { - "name": "brightPattern", + "name": "imgPattern", "type": "string", - "description": "Filename pattern used to match brightfield files to image files", + "description": "Filename pattern used to separate data and match with flatfied files", "required": true }, { - "name": "imgDir", + "name": "ffDir", "type": "collection", - "description": "Input image collection to be processed by this plugin", + "description": "Image collection containing flatfield and/or darkfield images", "required": true }, { - "name": "imgPattern", + "name": "ffPattern", "type": "string", - "description": "Filename pattern used to separate data and match with flatfied files", + "description": "Filename pattern used to match flatfield files to image files", "required": true }, { - "name": "photoPattern", + "name": "dfPattern", "type": "string", - "description": "Filename pattern used to match photobleach files to image files", - "required": true + "description": "Filename pattern used to match darkfield files to image files", + "required": false + }, + { + "name": "preview", + "type": "boolean", + "description": "Preview the output images' names without actually running computation", + "required": false } ], "outputs": [ @@ -68,7 +68,7 @@ { "key": "inputs.imgPattern", "title": "Image pattern", - "description": "Filename pattern used to separate data and match with flatfied files" + "description": "Filename pattern used to separate data and match with flatfield files" }, { "key": "inputs.ffDir", @@ -76,19 +76,19 @@ "description": "Image collection containing flatfield and/or darkfield images" }, { - "key": "inputs.brightPattern", - "title": "Brightfield file pattern", - "description": "Filename pattern used to match brightfield files to image files" + "key": "inputs.ffPattern", + "title": "Flatfield file pattern", + "description": "Filename pattern used to match flatfield files to image files" }, { - "key": "inputs.darkPattern", + "key": "inputs.dfPattern", "title": "Darkfield file pattern", "description": "Filename pattern used to match darkfield files to image files" }, { - "key": "inputs.photoPattern", - "title": "Photobleach file pattern", - "description": "Filename pattern used to match photobleach files to image files" + "key": "inputs.preview", + "title": "Preview Output", + "description": "Preview the output images' names without actually running computation" } ] } From c062ee47e475b41be6ef4c0c200d8f0ae5b0ec4f Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 21 Dec 2023 13:57:45 -0500 Subject: [PATCH 07/10] fixed env var in run-plugin script --- transforms/images/apply-flatfield-plugin/run-plugin.sh | 4 ++-- .../plugins/transforms/images/apply_flatfield/utils.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/transforms/images/apply-flatfield-plugin/run-plugin.sh b/transforms/images/apply-flatfield-plugin/run-plugin.sh index 8509f25ce..d42865d83 100755 --- a/transforms/images/apply-flatfield-plugin/run-plugin.sh +++ b/transforms/images/apply-flatfield-plugin/run-plugin.sh @@ -15,10 +15,10 @@ dfPattern="p{p:d+}_x\\(01-24\\)_y\(01-16\\)_wx\\(1-3\\)_wy\\(1-3\\)_c{c:d+}_dark # Output paths outDir=/data/outputs -FILE_EXT=".ome.zarr" +POLUS_IMG_EXT=".ome.zarr" docker run --mount type=bind,source=${datapath},target=/data/ \ - -e POLUS_EXT=${FILE_EXT} \ + -e POLUS_IMG_EXT=${POLUS_IMG_EXT} \ --user $(id -u):$(id -g) \ polusai/apply-flatfield-plugin:${version} \ --imgDir ${imgDir} \ diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/utils.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/utils.py index 246c9372f..d32e577a1 100644 --- a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/utils.py +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/utils.py @@ -41,7 +41,11 @@ def save_img( image: image to be saved out_dir: directory to save image """ - out_path = out_dir / inp_path.name + out_stem = inp_path.stem + if ".ome" in out_stem: + out_stem = out_stem.split(".ome")[0] + + out_path = out_dir / f"{out_stem}{POLUS_IMG_EXT}" with bfio.BioReader(inp_path, MAX_WORKERS) as reader, bfio.BioWriter( out_path, MAX_WORKERS, From c7ddacc8cd529589c35bbbe86d6ccb8e3aebaae7 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Thu, 21 Dec 2023 13:59:08 -0500 Subject: [PATCH 08/10] using preadator instead of built-in executor --- .../apply-flatfield-plugin/pyproject.toml | 1 + .../images/apply_flatfield/apply_flatfield.py | 40 ++++++++----------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/transforms/images/apply-flatfield-plugin/pyproject.toml b/transforms/images/apply-flatfield-plugin/pyproject.toml index a8f1b3223..588adec95 100644 --- a/transforms/images/apply-flatfield-plugin/pyproject.toml +++ b/transforms/images/apply-flatfield-plugin/pyproject.toml @@ -16,6 +16,7 @@ filepattern = "^2.0.4" typer = { version = "^0.7.0", extras = ["all"] } numpy = "^1.24.3" tqdm = "^4.65.0" +preadator = "0.4.0-dev2" [tool.poetry.group.dev.dependencies] bump2version = "^1.0.1" diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py index 32ddca010..d139430b2 100644 --- a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py @@ -1,6 +1,5 @@ """Provides the function to apply flatfield.""" -import concurrent.futures import logging import operator import pathlib @@ -8,6 +7,7 @@ import bfio import numpy +import preadator import tqdm from filepattern import FilePattern @@ -141,20 +141,19 @@ def _unshade_batch( df_image: component to be used for flatfield correction """ # Load images - images = [] - with concurrent.futures.ProcessPoolExecutor( - max_workers=utils.MAX_WORKERS, + with preadator.ProcessManager( + name="unshade_batch::load", + num_processes=utils.MAX_WORKERS, + threads_per_process=2, ) as load_executor: load_futures = [] for i, inp_path in enumerate(batch_paths): - load_futures.append(load_executor.submit(utils.load_img, inp_path, i)) + load_futures.append( + load_executor.submit_process(utils.load_img, inp_path, i), + ) - for lf in tqdm.tqdm( - concurrent.futures.as_completed(load_futures), - total=len(load_futures), - desc="Loading batch", - ): - images.append(lf.result()) + load_executor.join_processes() + images = [f.result() for f in load_futures] images = [img for _, img in sorted(images, key=operator.itemgetter(0))] img_stack = numpy.stack(images, axis=0) @@ -166,18 +165,11 @@ def _unshade_batch( img_stack /= ff_image + 1e-8 # Save outputs - with concurrent.futures.ProcessPoolExecutor( - max_workers=utils.MAX_WORKERS, + with preadator.ProcessManager( + name="unshade_batch::save", + num_processes=utils.MAX_WORKERS, + threads_per_process=2, ) as save_executor: - save_futures = [] for inp_path, img in zip(batch_paths, img_stack): - save_futures.append( - save_executor.submit(utils.save_img, inp_path, img, out_dir), - ) - - for sf in tqdm.tqdm( - concurrent.futures.as_completed(save_futures), - total=len(save_futures), - desc="Saving batch", - ): - sf.result() + save_executor.submit_process(utils.save_img, inp_path, img, out_dir) + save_executor.join_processes() From d422a6eace50110e47923dda3c904867596a61c4 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Fri, 22 Dec 2023 17:28:35 -0500 Subject: [PATCH 09/10] fix: casting original images to numpy.float32 --- .../transforms/images/apply_flatfield/apply_flatfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py index d139430b2..40e895a1c 100644 --- a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py @@ -156,7 +156,7 @@ def _unshade_batch( images = [f.result() for f in load_futures] images = [img for _, img in sorted(images, key=operator.itemgetter(0))] - img_stack = numpy.stack(images, axis=0) + img_stack = numpy.stack(images, axis=0).astype(numpy.float32) # Apply flatfield correction if df_image is not None: From f5677281b3ad27f49d1c67906b5166731221a1e2 Mon Sep 17 00:00:00 2001 From: Najib Ishaq Date: Fri, 22 Dec 2023 17:34:23 -0500 Subject: [PATCH 10/10] chore: better logging --- .../transforms/images/apply_flatfield/apply_flatfield.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py index 40e895a1c..d81ef71da 100644 --- a/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py +++ b/transforms/images/apply-flatfield-plugin/src/polus/plugins/transforms/images/apply_flatfield/apply_flatfield.py @@ -101,6 +101,10 @@ def _unshade_images( ff_path: path to the flatfield image df_path: path to the darkfield image """ + logger.info(f"Applying flatfield correction to {len(img_paths)} images ...") + logger.info(f"{ff_path.name = } ...") + logger.debug(f"Images: {img_paths}") + with bfio.BioReader(ff_path, max_workers=2) as bf: ff_image = bf[:, :, :, 0, 0].squeeze()