diff --git a/formats/ome-converter-tool/.bumpversion.cfg b/formats/ome-converter-tool/.bumpversion.cfg index bf31b9cf5..c2255d773 100644 --- a/formats/ome-converter-tool/.bumpversion.cfg +++ b/formats/ome-converter-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.2 +current_version = 0.3.3-dev3 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? @@ -24,6 +24,10 @@ replace = version = "{new_version}" [bumpversion:file:VERSION] +[bumpversion:file:ict.yaml] + +[bumpversion:file:omeconverter.cwl] + [bumpversion:file:README.md] [bumpversion:file:src/polus/images/formats/ome_converter/__init__.py] diff --git a/formats/ome-converter-tool/Dockerfile b/formats/ome-converter-tool/Dockerfile index 088c8b643..53a844554 100644 --- a/formats/ome-converter-tool/Dockerfile +++ b/formats/ome-converter-tool/Dockerfile @@ -1,4 +1,4 @@ -FROM polusai/bfio:2.3.3 +FROM polusai/bfio:2.4.4 # environment variables defined in polusai/bfio ENV EXEC_DIR="/opt/executables" diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index 3fbcc8491..1eb47cbab 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -1,4 +1,4 @@ -# OME Converter (v0.3.2) +# OME Converter (v0.3.3-dev3) This WIPP plugin converts BioFormats supported data types to the OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file @@ -23,12 +23,16 @@ contents of `plugin.json` into the pop-up window and submit. ## Options -This plugin takes 3 input arguments and 1 output argument: +This plugin takes 2 input arguments and 1 output argument: | Name | Description | I/O | Type | |------------------|--------------------------------------------------------------|--------|-------------| | `--inpDir` | Input generic data collection to be processed by this plugin | Input | genericData | | `--filePattern` | A filepattern, used to select data for conversion | Input | string | -| `--fileExtension`| A desired file format for conversion | Input | enum | | `--outDir` | Output collection | Output | genericData | | `--preview` | Generate a JSON file with outputs | Output | JSON | + +## Docker Command + +```bash +docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.3-dev3 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs diff --git a/formats/ome-converter-tool/VERSION b/formats/ome-converter-tool/VERSION index d15723fbe..f3f8731e3 100644 --- a/formats/ome-converter-tool/VERSION +++ b/formats/ome-converter-tool/VERSION @@ -1 +1 @@ -0.3.2 +0.3.3-dev3 diff --git a/formats/ome-converter-tool/ict.yaml b/formats/ome-converter-tool/ict.yaml index 63658d64c..f6b3688de 100644 --- a/formats/ome-converter-tool/ict.yaml +++ b/formats/ome-converter-tool/ict.yaml @@ -1,55 +1,41 @@ author: -- Nick Schaub -- Hamdah Shafqat + - Nick Schaub + - Hamdah Shafqat contact: nick.schaub@nih.gov -container: polusai/ome-converter-tool:0.3.2-dev0 +container: polusai/ome-converter-tool:0.3.3-dev3 description: Convert Bioformats supported format to OME Zarr or OME TIF entrypoint: python3 -m polus.images.formats.ome_converter inputs: -- description: Input generic data collection to be processed by this plugin - format: - - genericData - name: inpDir - required: true - type: path -- description: A filepattern, used to select data to be converted - format: - - string - name: filePattern - required: true - type: string -- description: Type of data conversion - format: - - enum - name: fileExtension - required: true - type: string + - description: Input generic data collection to be processed by this plugin + format: + - genericData + name: inpDir + required: true + type: path + - description: A filepattern, used to select data to be converted + format: + - string + name: filePattern + required: true + type: string name: polusai/OMEConverter outputs: -- description: Output collection - format: - - genericData - name: outDir - required: true - type: path -repository: https://github.com/PolusAI/polus-plugins + - description: Output collection + format: + - genericData + name: outDir + required: true + type: path +repository: https://github.com/PolusAI/image-tools specVersion: 1.0.0 title: OME Converter ui: -- description: Input generic data collection to be processed by this plugin - key: inputs.inpDir - title: Input generic collection - type: path -- description: A filepattern, used to select data for conversion - key: inputs.filePattern - title: Filepattern - type: text -- description: Type of data conversion - fields: - - .ome.tif - - .ome.zarr - - default - key: inputs.fileExtension - title: fileExtension - type: select -version: 0.3.2-dev0 + - description: Input generic data collection to be processed by this plugin + key: inputs.inpDir + title: Input generic collection + type: path + - description: A filepattern, used to select data for conversion + key: inputs.filePattern + title: Filepattern + type: text +version: 0.3.3-dev3 diff --git a/formats/ome-converter-tool/omeconverter.cwl b/formats/ome-converter-tool/omeconverter.cwl index e242d5409..42340d71b 100644 --- a/formats/ome-converter-tool/omeconverter.cwl +++ b/formats/ome-converter-tool/omeconverter.cwl @@ -1,10 +1,6 @@ class: CommandLineTool cwlVersion: v1.2 inputs: - fileExtension: - inputBinding: - prefix: --fileExtension - type: string filePattern: inputBinding: prefix: --filePattern @@ -24,7 +20,7 @@ outputs: type: Directory requirements: DockerRequirement: - dockerPull: polusai/ome-converter-tool:0.3.2-dev0 + dockerPull: polusai/ome-converter-tool:0.3.3-dev3 InitialWorkDirRequirement: listing: - entry: $(inputs.outDir) diff --git a/formats/ome-converter-tool/plugin.json b/formats/ome-converter-tool/plugin.json index b00ef8446..9089f13f1 100644 --- a/formats/ome-converter-tool/plugin.json +++ b/formats/ome-converter-tool/plugin.json @@ -1,14 +1,14 @@ { "name": "OME Converter", - "version": "0.3.2", + "version": "0.3.3-dev3", "title": "OME Converter", "description": "Convert Bioformats supported format to OME Zarr or OME TIF", "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", "institution": "National Center for Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/PolusAI/polus-plugins", + "repository": "https://github.com/PolusAI/image-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/ome-converter-tool:0.3.2", + "containerId": "polusai/ome-converter-tool:0.3.3-dev3", "baseCommand": [ "python3", "-m", @@ -26,20 +26,6 @@ "type": "string", "description": "A filepattern, used to select data to be converted", "required": true - }, - { - "name": "fileExtension", - "type": "enum", - "description": "Type of data conversion", - "default": "default", - "options": { - "values": [ - ".ome.tif", - ".ome.zarr", - "default" - ] - }, - "required": true } ], "outputs": [ @@ -59,12 +45,6 @@ "key": "inputs.filePattern", "title": "Filepattern", "description": "A filepattern, used to select data for conversion" - }, - { - "key": "inputs.fileExtension", - "title": "fileExtension", - "description": "Type of data conversion", - "default": ".ome.tif" } ] } diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index ef070ca62..4866564cb 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-formats-ome-converter" -version = "0.3.2" +version = "0.3.3-dev3" description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format" authors = [ "Nick Schaub ", @@ -12,7 +12,7 @@ packages = [{include = "polus", from = "src"}] [tool.poetry.dependencies] python = ">=3.9,<3.12" -bfio = {version = "^2.3.3", extras = ["all"]} +bfio = {version = "^2.4.4", extras = ["all"]} filepattern = "^2.0.4" typer = "^0.7.0" tqdm = "^4.64.1" diff --git a/formats/ome-converter-tool/run-plugin.sh b/formats/ome-converter-tool/run-plugin.sh index 154496d8b..b579515f9 100755 --- a/formats/ome-converter-tool/run-plugin.sh +++ b/formats/ome-converter-tool/run-plugin.sh @@ -12,12 +12,11 @@ fileExtension=".ome.zarr" outDir=/data/output # Show the help options -docker run polusai/ome-converter-plugin:${version} +# docker run polusai/ome-converter-plugin:${version} # Run the plugin -docker run --mount type=bind,source=${datapath},target=/data/ \ - polusai/ome-converter-plugin:${version} \ +docker run -e POLUS_IMG_EXT=${fileExtension} --mount type=bind,source=${datapath},target=/data/ \ + polusai/ome-converter-tool:${version} \ --inpDir ${inpDir} \ --filePattern ${filePattern} \ - --fileExtension ${fileExtension} \ --outDir ${outDir} diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py index 601dd3281..941a41f60 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py @@ -1,6 +1,6 @@ """Ome Converter.""" -__version__ = "0.3.2" +__version__ = "0.3.3-dev3" from .image_converter import batch_convert from .image_converter import convert_image diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py index 7c4c36559..969c803ff 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py @@ -11,7 +11,6 @@ import preadator import typer from polus.images.formats.ome_converter.image_converter import NUM_THREADS -from polus.images.formats.ome_converter.image_converter import Extension from polus.images.formats.ome_converter.image_converter import convert_image from tqdm import tqdm @@ -24,6 +23,7 @@ ) logger = logging.getLogger("polus.images.formats.ome_converter") logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO)) +POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") @app.command() @@ -43,11 +43,6 @@ def main( "--filePattern", help="A filepattern defining the images to be converted", ), - file_extension: Extension = typer.Option( - Extension, - "--fileExtension", - help="Type of data conversion", - ), out_dir: pathlib.Path = typer.Option( ..., "--outDir", @@ -68,7 +63,6 @@ def main( logger.info(f"inpDir = {inp_dir}") logger.info(f"outDir = {out_dir}") logger.info(f"filePattern = {pattern}") - logger.info(f"fileExtension = {file_extension}") fps = fp.FilePattern(inp_dir, pattern) @@ -79,7 +73,7 @@ def main( "outDir": [], } for file in fps(): - out_name = str(file[1][0].name.split(".")[0]) + file_extension + out_name = str(file[1][0].name.split(".")[0]) + POLUS_IMG_EXT out_json["outDir"].append(out_name) json.dump(out_json, jfile, indent=2) return @@ -93,14 +87,14 @@ def main( for files in fps(): file = files[1][0] threads.append( - executor.submit_process(convert_image, file, file_extension, out_dir), + executor.submit_process(convert_image, file, POLUS_IMG_EXT, out_dir), ) for f in tqdm( as_completed(threads), total=len(threads), mininterval=5, - desc=f"converting images to {file_extension}", + desc=f"converting images to {POLUS_IMG_EXT}", initial=0, unit_scale=True, colour="cyan", diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index 8bc5f174a..825df42b9 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -1,14 +1,15 @@ """Ome Converter.""" -import enum + import logging import os import pathlib from concurrent.futures import as_completed -from multiprocessing import cpu_count +from itertools import product from sys import platform from typing import Optional import filepattern as fp +import numpy as np import preadator from bfio import BioReader from bfio import BioWriter @@ -20,25 +21,55 @@ TILE_SIZE = 2**13 -if platform == "linux" or platform == "linux2": +if platform.startswith("linux"): NUM_THREADS = len(os.sched_getaffinity(0)) // 2 # type: ignore else: - NUM_THREADS = max(cpu_count() // 2, 1) + NUM_THREADS = os.cpu_count() # type: ignore -POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") +def write_image( + br: BioReader, + c: int, + image: np.ndarray, + out_path: pathlib.Path, + max_workers: int, +) -> None: + """Write an image to OME-TIFF or OME-ZARR format using BioFormats. -class Extension(str, enum.Enum): - """Extension types to be converted.""" + This function converts a given image to the OME-TIFF or OME-ZARR format, + ensuring that the data type is compatible and handling any necessary + byte order adjustments. It utilizes the BioWriter for writing the image + and manages channel names based on the provided BioReader. - OMETIF = ".ome.tif" - OMEZARR = ".ome.zarr" - Default = POLUS_IMG_EXT + Args: + br: An instance of BioReader containing metadata for the image. + c: The index of the channel in the image. + image: The image data to be written. + out_path: Path to an output image. + max_workers: The maximum number of worker threads to use for writing. + """ + if image.dtype == ">u2": + image = image.byteswap().newbyteorder("<") + + with BioWriter( + out_path, + max_workers, + ) as bw: + # Handling of parsing channels when channels names are not provided. + if bw.channel_names != [None]: + bw.channel_names = [br.channel_names[c]] + bw.C = 1 + bw.T = 1 + bw.Z = 1 + bw.X = image.shape[1] + bw.Y = image.shape[0] + bw.dtype = image.dtype + bw[:] = image def convert_image( inp_image: pathlib.Path, - file_extension: Extension, + file_extension: str, out_dir: pathlib.Path, ) -> None: """Convert bioformats supported datatypes to ome.tif or ome.zarr file format. @@ -48,78 +79,75 @@ def convert_image( file_extension: Type of data conversion. out_dir: Path to output directory. """ - with BioReader(inp_image, max_workers=2) as br: - # Loop through timepoints - for t in range(br.T): - # Loop through channels - for c in range(br.C): - extension = "".join( - [ - suffix - for suffix in inp_image.suffixes[-2:] - if len(suffix) < 6 # noqa: PLR2004 - ], + # Loop through timepoints, channels and z-slices + with BioReader(inp_image, max_workers=NUM_THREADS) as br: + for t, c, z in product(range(br.T), range(br.C), range(br.Z)): + extension = "".join( + [ + suffix + for suffix in inp_image.suffixes[-2:] + if len(suffix) < 6 # noqa: PLR2004 + ], + ) + + out_path = out_dir.joinpath( + inp_image.name.replace(extension, file_extension), + ) + if br.C > 1: + out_path = out_dir.joinpath( + out_path.name.replace( + file_extension, + f"_c{c}" + file_extension, + ), + ) + if br.T > 1: + out_path = out_dir.joinpath( + out_path.name.replace( + file_extension, + f"_t{t}" + file_extension, + ), ) + if br.Z > 1: out_path = out_dir.joinpath( - inp_image.name.replace(extension, file_extension), + out_path.name.replace( + file_extension, + f"_z{z}" + file_extension, + ), ) - if br.C > 1: - out_path = out_dir.joinpath( - out_path.name.replace( - file_extension, - f"_c{c}" + file_extension, - ), - ) - if br.T > 1: - out_path = out_dir.joinpath( - out_path.name.replace( - file_extension, - f"_t{t}" + file_extension, - ), - ) - - with BioWriter( - out_path, - max_workers=2, - metadata=br.metadata, - ) as bw: - bw.C = 1 - bw.T = 1 - - # Handling of parsing channels when channels names are not provided. - if bw.channel_names != [None]: - bw.channel_names = [br.channel_names[c]] - - # Loop through z-slices - for z in range(br.Z): - # Loop across the length of the image - for y in range(0, br.Y, TILE_SIZE): - y_max = min([br.Y, y + TILE_SIZE]) - - # Loop across the depth of the image - for x in range(0, br.X, TILE_SIZE): - x_max = min([br.X, x + TILE_SIZE]) - bw[ - y:y_max, - x:x_max, - z : z + 1, - 0, - 0, - ] = br[ - y:y_max, - x:x_max, - z : z + 1, - c, - t, - ] + + # Initialize the complete_image if not done yet + final_image = np.zeros((br.Y, br.X, 1, 1, 1), dtype=br.dtype) + + # Process each tile in the image using itertools.product + for y, x in product(range(0, br.Y, TILE_SIZE), range(0, br.X, TILE_SIZE)): + y_max = min(br.Y, y + TILE_SIZE) + x_max = min(br.X, x + TILE_SIZE) + + image = br[ + y:y_max, + x:x_max, + z, + c, + t, + ] + # Place the tile into the correct position in the complete image + final_image[y:y_max, x:x_max, z, c, t] = image + + write_image( + br=br, + c=c, + image=final_image, + out_path=out_path, + max_workers=NUM_THREADS, + ) def batch_convert( inp_dir: pathlib.Path, out_dir: pathlib.Path, file_pattern: Optional[str], - file_extension: Extension, + file_extension: str, ) -> None: """Convert bioformats supported datatypes in batches to ome.tif or ome.zarr. diff --git a/formats/ome-converter-tool/tests/conftest.py b/formats/ome-converter-tool/tests/conftest.py index 4b1931a57..b3dc8a7b9 100644 --- a/formats/ome-converter-tool/tests/conftest.py +++ b/formats/ome-converter-tool/tests/conftest.py @@ -10,7 +10,6 @@ import pytest import requests import skimage.data -import skimage.io import skimage.measure @@ -52,15 +51,18 @@ def synthetic_images( size, extension = get_params syn_dir = pathlib.Path(tempfile.mkdtemp(suffix="_syn_data")) + images: list[numpy.ndarray] = [] - for i in range(10): - # Create images - blobs: numpy.ndarray = skimage.data.binary_blobs( - length=size, - volume_fraction=0.05, - blob_size_fraction=0.05, + # Create a random number generator + rng = numpy.random.default_rng() + + for i in range(3): + syn_img: numpy.ndarray = rng.integers( + low=0, + high=256, + size=(size, size), + dtype=numpy.uint8, ) - syn_img: numpy.ndarray = skimage.measure.label(blobs) outname = f"syn_image_{i}{extension}" # Save image diff --git a/formats/ome-converter-tool/tests/test_main.py b/formats/ome-converter-tool/tests/test_main.py index 027a4a1a6..8e91d9480 100644 --- a/formats/ome-converter-tool/tests/test_main.py +++ b/formats/ome-converter-tool/tests/test_main.py @@ -59,7 +59,6 @@ def test_image_converter_omezarr( def test_cli( synthetic_images: tuple[list[np.ndarray], pathlib.Path], output_directory: pathlib.Path, - file_extension: str, ) -> None: """Test Cli.""" _, inp_dir = synthetic_images @@ -71,8 +70,6 @@ def test_cli( inp_dir, "--filePattern", ".+", - "--fileExtension", - file_extension, "--outDir", output_directory, ],