diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 33779da..5f0daa4 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -40,7 +40,7 @@ jobs: run: python -m pytest -vv -s ./tests/test_macenko_module - name : Run Core Tests run: python -m pytest -vv -s ./tests/test_core_modules - - name : Run Core Tests + - name : Run Dicom Tests run: python -m pytest -vv -s ./tests/test_dicom_module - name : Run Dataset and Dataloader Tests run: python -m pytest -vv -s ./tests/test_pytorch_dataset diff --git a/PathoPatch.ipynb b/PathoPatch.ipynb index ad0ad65..744f136 100644 --- a/PathoPatch.ipynb +++ b/PathoPatch.ipynb @@ -1,15 +1,5 @@ { "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "view-in-github" - }, - "source": [ - "\"Open" - ] - }, { "cell_type": "markdown", "metadata": { @@ -36,32 +26,69 @@ "base_uri": "https://localhost:8080/" }, "id": "syrpNbj1jYDH", - "outputId": "3aab3f77-360c-4537-a341-22e57bda0d4a" + "outputId": "00b06be6-5d1f-4fcd-a997-edeab2d0e88c" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33m\r0% [Working]\u001b[0m\r \rHit:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease\n", - "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 InRelease\n", - "Hit:3 http://security.ubuntu.com/ubuntu jammy-security InRelease\n", + "\u001b[33m\r0% [Working]\u001b[0m\r \rGet:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n", + "\u001b[33m\r0% [Connecting to archive.ubuntu.com] [Waiting for headers] [1 InRelease 3,626 B/3,626 B 100%] [Conn\u001b[0m\r \rGet:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB]\n", + "Hit:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 InRelease\n", "Hit:4 http://archive.ubuntu.com/ubuntu jammy InRelease\n", - "Hit:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease\n", + "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB]\n", "Hit:6 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n", "Hit:7 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease\n", "Hit:8 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n", "Hit:9 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n", "Hit:10 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n", + "Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,920 kB]\n", + "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,104 kB]\n", + "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,358 kB]\n", + "Fetched 5,615 kB in 2s (2,454 kB/s)\n", "Reading package lists... Done\n", "Building dependency tree... Done\n", "Reading state information... Done\n", - "39 packages can be upgraded. Run 'apt list --upgradable' to see them.\n", + "45 packages can be upgraded. Run 'apt list --upgradable' to see them.\n", "Reading package lists... Done\n", "Building dependency tree... Done\n", "Reading state information... Done\n", - "openslide-tools is already the newest version (3.4.1+dfsg-5build1).\n", - "0 upgraded, 0 newly installed, 0 to remove and 39 not upgraded.\n" + "The following additional packages will be installed:\n", + " libopenslide0\n", + "Suggested packages:\n", + " libtiff-tools\n", + "The following NEW packages will be installed:\n", + " libopenslide0 openslide-tools\n", + "0 upgraded, 2 newly installed, 0 to remove and 45 not upgraded.\n", + "Need to get 104 kB of archives.\n", + "After this operation, 297 kB of additional disk space will be used.\n", + "Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libopenslide0 amd64 3.4.1+dfsg-5build1 [89.8 kB]\n", + "Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 openslide-tools amd64 3.4.1+dfsg-5build1 [13.8 kB]\n", + "Fetched 104 kB in 0s (416 kB/s)\n", + "Selecting previously unselected package libopenslide0.\n", + "(Reading database ... 121753 files and directories currently installed.)\n", + "Preparing to unpack .../libopenslide0_3.4.1+dfsg-5build1_amd64.deb ...\n", + "Unpacking libopenslide0 (3.4.1+dfsg-5build1) ...\n", + "Selecting previously unselected package openslide-tools.\n", + "Preparing to unpack .../openslide-tools_3.4.1+dfsg-5build1_amd64.deb ...\n", + "Unpacking openslide-tools (3.4.1+dfsg-5build1) ...\n", + "Setting up libopenslide0 (3.4.1+dfsg-5build1) ...\n", + "Setting up openslide-tools (3.4.1+dfsg-5build1) ...\n", + "Processing triggers for man-db (2.10.2-1) ...\n", + "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n", + "\n" ] } ], @@ -71,87 +98,114 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "WkCsLuQAj41e", - "outputId": "796bb818-c29c-4a22-df88-9eecebc897ff" + "outputId": "a8fed62a-3989-44a8-c20b-f1a757f32c92" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: openslide-python in /usr/local/lib/python3.10/site-packages (1.3.1)\n", - "Requirement already satisfied: pathopatch in /usr/local/lib/python3.10/site-packages (0.9.5.1b0)\n", - "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/site-packages (from openslide-python) (10.2.0)\n", - "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/site-packages (from pathopatch) (6.0.1)\n", - "Requirement already satisfied: Shapely==1.8.5.post1 in /usr/local/lib/python3.10/site-packages (from pathopatch) (1.8.5.post1)\n", - "Requirement already satisfied: colorama in /usr/local/lib/python3.10/site-packages (from pathopatch) (0.4.6)\n", - "Requirement already satisfied: future in /usr/local/lib/python3.10/site-packages (from pathopatch) (1.0.0)\n", - "Requirement already satisfied: geojson>=3.0.0 in /usr/local/lib/python3.10/site-packages (from pathopatch) (3.1.0)\n", - "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/site-packages (from pathopatch) (3.8.3)\n", - "Requirement already satisfied: natsort in /usr/local/lib/python3.10/site-packages (from pathopatch) (8.4.0)\n", - "Requirement already satisfied: numpy<1.24,>1.22 in /usr/local/lib/python3.10/site-packages (from pathopatch) (1.23.5)\n", - "Requirement already satisfied: opencv-python-headless in /usr/local/lib/python3.10/site-packages (from pathopatch) (4.9.0.80)\n", - "Requirement already satisfied: pandas in /usr/local/lib/python3.10/site-packages (from pathopatch) (2.2.1)\n", - "Requirement already satisfied: pydantic==1.10.4 in /usr/local/lib/python3.10/site-packages (from pathopatch) (1.10.4)\n", - "Requirement already satisfied: rasterio==1.3.5.post1 in /usr/local/lib/python3.10/site-packages (from pathopatch) (1.3.5.post1)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/site-packages (from pathopatch) (2.31.0)\n", - "Requirement already satisfied: scikit-image in /usr/local/lib/python3.10/site-packages (from pathopatch) (0.22.0)\n", - "Requirement already satisfied: setuptools<=65.6.3 in /usr/local/lib/python3.10/site-packages (from pathopatch) (65.6.3)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/site-packages (from pathopatch) (4.66.1)\n", - "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/site-packages (from pathopatch) (0.17.1)\n", - "Requirement already satisfied: torch in /usr/local/lib/python3.10/site-packages (from pathopatch) (2.2.1)\n", - "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/site-packages (from pydantic==1.10.4->pathopatch) (4.10.0)\n", - "Requirement already satisfied: affine in /usr/local/lib/python3.10/site-packages (from rasterio==1.3.5.post1->pathopatch) (2.4.0)\n", - "Requirement already satisfied: attrs in /usr/local/lib/python3.10/site-packages (from rasterio==1.3.5.post1->pathopatch) (23.2.0)\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.10/site-packages (from rasterio==1.3.5.post1->pathopatch) (2023.11.17)\n", - "Requirement already satisfied: click>=4.0 in /usr/local/lib/python3.10/site-packages (from rasterio==1.3.5.post1->pathopatch) (8.1.7)\n", - "Requirement already satisfied: cligj>=0.5 in /usr/local/lib/python3.10/site-packages (from rasterio==1.3.5.post1->pathopatch) (0.7.2)\n", - "Requirement already satisfied: snuggs>=1.4.1 in /usr/local/lib/python3.10/site-packages (from rasterio==1.3.5.post1->pathopatch) (1.4.7)\n", - "Requirement already satisfied: click-plugins in /usr/local/lib/python3.10/site-packages (from rasterio==1.3.5.post1->pathopatch) (1.1.1)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->pathopatch) (1.2.0)\n", - "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/site-packages (from matplotlib->pathopatch) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/site-packages (from matplotlib->pathopatch) (4.50.0)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->pathopatch) (1.4.5)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib->pathopatch) (23.2)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->pathopatch) (3.1.2)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/site-packages (from matplotlib->pathopatch) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas->pathopatch) (2024.1)\n", - "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/site-packages (from pandas->pathopatch) (2024.1)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests->pathopatch) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests->pathopatch) (3.6)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/site-packages (from requests->pathopatch) (2.1.0)\n", - "Requirement already satisfied: scipy>=1.8 in /usr/local/lib/python3.10/site-packages (from scikit-image->pathopatch) (1.12.0)\n", - "Requirement already satisfied: networkx>=2.8 in /usr/local/lib/python3.10/site-packages (from scikit-image->pathopatch) (3.2.1)\n", - "Requirement already satisfied: imageio>=2.27 in /usr/local/lib/python3.10/site-packages (from scikit-image->pathopatch) (2.34.0)\n", - "Requirement already satisfied: tifffile>=2022.8.12 in /usr/local/lib/python3.10/site-packages (from scikit-image->pathopatch) (2024.2.12)\n", - "Requirement already satisfied: lazy_loader>=0.3 in /usr/local/lib/python3.10/site-packages (from scikit-image->pathopatch) (0.3)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (3.13.3)\n", - "Requirement already satisfied: sympy in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (1.12)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (3.1.3)\n", - "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (2024.3.1)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (12.1.105)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (8.9.2.26)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (12.1.3.1)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (11.0.2.54)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (10.3.2.106)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (11.4.5.107)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (12.1.0.106)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (2.19.3)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (12.1.105)\n", - "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/site-packages (from torch->pathopatch) (2.2.0)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->pathopatch) (12.4.99)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib->pathopatch) (1.16.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/site-packages (from jinja2->torch->pathopatch) (2.1.5)\n", - "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/site-packages (from sympy->torch->pathopatch) (1.3.0)\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m" + "Collecting openslide-python\n", + " Downloading openslide-python-1.3.1.tar.gz (358 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m359.0/359.0 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Collecting pathopatch\n", + " Downloading pathopatch-1.0.0b0-py3-none-any.whl (83 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.6/83.6 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from openslide-python) (9.4.0)\n", + "Collecting Pillow (from openslide-python)\n", + " Downloading pillow-10.3.0-cp310-cp310-manylinux_2_28_x86_64.whl (4.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from pathopatch) (6.0.1)\n", + "Collecting Shapely==1.8.5.post1 (from pathopatch)\n", + " Downloading Shapely-1.8.5.post1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting colorama (from pathopatch)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from pathopatch) (0.18.3)\n", + "Collecting geojson>=3.0.0 (from pathopatch)\n", + " Downloading geojson-3.1.0-py3-none-any.whl (15 kB)\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from pathopatch) (3.7.1)\n", + "Requirement already satisfied: natsort in /usr/local/lib/python3.10/dist-packages (from pathopatch) (8.4.0)\n", + "Collecting numpy<1.24,>1.22 (from pathopatch)\n", + " Downloading numpy-1.23.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.1/17.1 MB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: opencv-python-headless in /usr/local/lib/python3.10/dist-packages (from pathopatch) (4.9.0.80)\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from pathopatch) (1.5.3)\n", + "Collecting pydantic==1.10.4 (from pathopatch)\n", + " Downloading pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting rasterio==1.3.5.post1 (from pathopatch)\n", + " Downloading rasterio-1.3.5.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (20.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.0/20.0 MB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from pathopatch) (2.31.0)\n", + "Requirement already satisfied: scikit-image in /usr/local/lib/python3.10/dist-packages (from pathopatch) (0.19.3)\n", + "Collecting setuptools<=65.6.3 (from pathopatch)\n", + " Downloading setuptools-65.6.3-py3-none-any.whl (1.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from pathopatch) (4.66.2)\n", + "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from pathopatch) (0.17.1+cu121)\n", + "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from pathopatch) (2.2.1+cu121)\n", + "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic==1.10.4->pathopatch) (4.10.0)\n", + "Collecting affine (from rasterio==1.3.5.post1->pathopatch)\n", + " Downloading affine-2.4.0-py3-none-any.whl (15 kB)\n", + "Requirement already satisfied: attrs in /usr/local/lib/python3.10/dist-packages (from rasterio==1.3.5.post1->pathopatch) (23.2.0)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from rasterio==1.3.5.post1->pathopatch) (2024.2.2)\n", + "Requirement already satisfied: click>=4.0 in /usr/local/lib/python3.10/dist-packages (from rasterio==1.3.5.post1->pathopatch) (8.1.7)\n", + "Requirement already satisfied: cligj>=0.5 in /usr/local/lib/python3.10/dist-packages (from rasterio==1.3.5.post1->pathopatch) (0.7.2)\n", + "Collecting snuggs>=1.4.1 (from rasterio==1.3.5.post1->pathopatch)\n", + " Downloading snuggs-1.4.7-py3-none-any.whl (5.4 kB)\n", + "Requirement already satisfied: click-plugins in /usr/local/lib/python3.10/dist-packages (from rasterio==1.3.5.post1->pathopatch) (1.1.1)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->pathopatch) (1.2.0)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->pathopatch) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->pathopatch) (4.50.0)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->pathopatch) (1.4.5)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->pathopatch) (24.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->pathopatch) (3.1.2)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->pathopatch) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->pathopatch) (2023.4)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->pathopatch) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->pathopatch) (3.6)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->pathopatch) (2.0.7)\n", + "Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from scikit-image->pathopatch) (1.11.4)\n", + "Requirement already satisfied: networkx>=2.2 in /usr/local/lib/python3.10/dist-packages (from scikit-image->pathopatch) (3.2.1)\n", + "Requirement already satisfied: imageio>=2.4.1 in /usr/local/lib/python3.10/dist-packages (from scikit-image->pathopatch) (2.31.6)\n", + "Requirement already satisfied: tifffile>=2019.7.26 in /usr/local/lib/python3.10/dist-packages (from scikit-image->pathopatch) (2024.2.12)\n", + "Requirement already satisfied: PyWavelets>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-image->pathopatch) (1.5.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->pathopatch) (3.13.3)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->pathopatch) (1.12)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->pathopatch) (3.1.3)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->pathopatch) (2023.6.0)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch->pathopatch)\n", + " Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch->pathopatch)\n", + " Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch->pathopatch)\n", + " Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch->pathopatch)\n", + " Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch->pathopatch)\n", + " Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch->pathopatch)\n", + " Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch->pathopatch)\n", + " Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch->pathopatch)\n", + " Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", + "\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m84.9/124.2 MB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:30\u001b[0m" ] } ], @@ -161,34 +215,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "xdsM1hLnkC1D", - "outputId": "e9209dc9-7de8-44dc-b0a2-64fa1c355ec5" + "id": "xdsM1hLnkC1D" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting cucim\n", - " Downloading cucim-23.10.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (43 kB)\n", - "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/43.1 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.1/43.1 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/site-packages (from cucim) (8.1.7)\n", - "Requirement already satisfied: numpy in /usr/local/lib/python3.10/site-packages (from cucim) (1.23.5)\n", - "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/site-packages (from cucim) (0.3)\n", - "Downloading cucim-23.10.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (9.7 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.7/9.7 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hInstalling collected packages: cucim\n", - "Successfully installed cucim-23.10.0\n", - "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", - "\u001b[0m" - ] - } - ], + "outputs": [], "source": [ "!pip install cucim" ] @@ -206,46 +237,26 @@ "cell_type": "code", "execution_count": null, "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ldwjSWvYmbic", - "outputId": "4f3994ce-6486-4c7b-9821-f767fcc98720" + "id": "ldwjSWvYmbic" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "mkdir: cannot create directory ‘wsi_data’: File exists\n", - "--2024-03-26 21:12:26-- https://openslide.cs.cmu.edu/download/openslide-testdata/Aperio/CMU-1-Small-Region.svs\n", - "Resolving openslide.cs.cmu.edu (openslide.cs.cmu.edu)... 128.2.209.93\n", - "Connecting to openslide.cs.cmu.edu (openslide.cs.cmu.edu)|128.2.209.93|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 1938955 (1.8M) [image/tiff]\n", - "Saving to: ‘./wsi_data/CMU-1-Small-Region.svs.1’\n", - "\n", - "CMU-1-Small-Region. 100%[===================>] 1.85M 478KB/s in 4.4s \n", - "\n", - "2024-03-26 21:12:32 (427 KB/s) - ‘./wsi_data/CMU-1-Small-Region.svs.1’ saved [1938955/1938955]\n", - "\n", - "--2024-03-26 21:12:32-- https://openslide.cs.cmu.edu/download/openslide-testdata/Aperio/CMU-1.svs\n", - "Resolving openslide.cs.cmu.edu (openslide.cs.cmu.edu)... 128.2.209.93\n", - "Connecting to openslide.cs.cmu.edu (openslide.cs.cmu.edu)|128.2.209.93|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 177552579 (169M) [image/tiff]\n", - "Saving to: ‘./wsi_data/CMU-1.svs’\n", - "\n", - "CMU-1.svs 36%[======> ] 61.51M 457KB/s eta 2m 52s " - ] - } - ], + "outputs": [], "source": [ "!mkdir wsi_data\n", "!wget --directory-prefix ./wsi_data https://openslide.cs.cmu.edu/download/openslide-testdata/Aperio/CMU-1-Small-Region.svs\n", "!wget --directory-prefix ./wsi_data https://openslide.cs.cmu.edu/download/openslide-testdata/Aperio/CMU-1.svs" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "3p86K0Qv0h80" + }, + "source": [ + "## 3. Simple Patch-Extraction\n", + "\n", + "Simple patch extraction" + ] + }, { "cell_type": "code", "execution_count": null, @@ -253,22 +264,62 @@ "id": "6smBlRYcmgPQ" }, "outputs": [], - "source": [] + "source": [ + "!wsi_extraction \\\n", + " --wsi_paths \\\n", + " --output_path \\\n", + " --wsi_extension svs \\\n", + " --downsample 1 \\\n", + " --processes 8 \\\n", + " --log_level debug \\\n", + " --overwrite True" + ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "_5_FnB560qtz" + }, + "source": [ + "## 4. DICOM-Files" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "is9CORMn0CWy" + }, "source": [ "DICOM-Files:\n", "\n", - "whole folder must be provided, please use the wsi_extension .dcm for loading wsi, or provide a .csv file as a filelist with path to the dicom folder" + "whole folder must be provided, please use the wsi_extension .dcm for loading wsi, or provide a .csv file as a filelist with path to the dicom folder. Files can be converted using `wsidicomizer` (https://github.com/imi-bigpicture/wsidicomizer)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wrQq4WpW0vD6" + }, + "outputs": [], + "source": [ + "!pip install wsidicomizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UFVhoBet1OWc" + }, + "outputs": [], + "source": [ + "!wsidicomizer -i -o" ] } ], "metadata": { "colab": { - "authorship_tag": "ABX9TyOSg5Tomy2ythze0d941UHb", - "include_colab_link": true, "provenance": [] }, "kernelspec": { diff --git a/README.md b/README.md index 19cb2b2..a1846c0 100644 --- a/README.md +++ b/README.md @@ -371,6 +371,9 @@ In our Pre-Processing pipeline, we are able to extract quadratic patches from de An example notebook is given [here](PathoPatch.ipynb): Open In Colab +### Example config files +Example config files for various use-cases can be found in the [test database](tests/static_test_files/preprocessing) (tests/static_test_files/preprocessing) + ### DICOM-conversion To convert WSI-Files into DICOM-Format, please follow [this documentation](docs/DICOM.md) diff --git a/environment.yaml b/environment.yaml index 614815d..41bb476 100644 --- a/environment.yaml +++ b/environment.yaml @@ -7,6 +7,7 @@ dependencies: - openslide=3.4.1 - pip=23.0 - python-javabridge + - libjpeg-turbo - pip: - Pillow>=9.5.0 - PyYAML