Merge branch 'main' into separate_nuc_wc

.github/scripts/get_example_dataset.py

@@ -1,33 +1,28 @@
-import pathlib
+from pathlib import Path
+import os
 
 import datasets
 
 datasets.disable_progress_bar()
 
-valid_datasets = [
-    "segment_image_data",
-    "cluster_pixels",
-    "cluster_cells",
-    "post_clustering",
-    "fiber_segmentation",
-    "LDA_preprocessing",
-    "LDA_training_inference",
-    "neighborhood_analysis",
-    "pairwise_spatial_enrichment",
-    "ome_tiff",
-]
+DATASET_PATH = "angelolab/ark_example"
 
-def load_dataset(cache_dir: pathlib.Path, name: str):
+valid_configs = datasets.get_dataset_config_names(DATASET_PATH)
+
+def load_dataset(cache_dir: Path, name: str):
     _ = datasets.load_dataset(
-        path="angelolab/ark_example",
+        path=DATASET_PATH,
         cache_dir=cache_dir,
         name=name,
-        use_auth_token=False,
-        revision="main"
+        token=False,
+        revision="main",
+        trust_remote_code=True,
     )
 
-# Make the cache directory if it doesn't exist.
-cache_dir = pathlib.Path("./data/cache/")
+# Create the cache directory
+cache_dir = Path(os.environ.get("GITHUB_WORKSPACE")).resolve() / "data" / "cache"
 cache_dir.mkdir(parents=True, exist_ok=True)
-for dataset_config in valid_datasets:
-    load_dataset(cache_dir=cache_dir.as_posix(), name=dataset_config)
+
+# Download all available datasets
+for dataset_config in valid_configs:
+    load_dataset(cache_dir=cache_dir, name=dataset_config)

.github/workflows/build.yml

@@ -14,39 +14,39 @@ jobs:
       # Ensure that a wheel builder finishes even if another fails
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-12]
+        os: [ubuntu-latest, windows-latest, macos-13]
 
     steps:
       - name: Checkout ${{ github.repository }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up QEMU (For Linux ARM)
         if: runner.os == 'Linux'
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
 
       - name: Build Wheels
-        uses: pypa/cibuildwheel@v2.13.0
+        uses: pypa/cibuildwheel@v2.16.5
         with:
           package-dir: .
           output-dir: wheelhouse
           config-file: "{package}/pyproject.toml"
 
       - name: Store Wheel Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: distributions
+          name: dist-${{ matrix.os }}
           path: wheelhouse/*.whl
 
   build_sdist:
     name:  Source Distribution
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ${{ github.repository }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build sdist
         run: pipx run build --sdist
@@ -55,7 +55,7 @@ jobs:
         run: pipx run twine check dist/*
 
       - name: Store sdist Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: distributions
+          name: dist-sdist
           path: dist/*.tar.gz

.github/workflows/ci.yml

@@ -25,20 +25,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ${{github.repository }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Restore Example Dataset Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: dataset-cache
         with:
           path: ./data/cache/
           key: huggingface-${{ env.cache-name }}
           enableCrossOsArchive: true
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
           cache-dependency-path: "**/pyproject.toml"
@@ -56,7 +56,7 @@ jobs:
       pull-requests: write
     secrets: inherit
     uses: ./.github/workflows/test.yml
-  
+
   build:
     name: Build
     permissions:
@@ -71,13 +71,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ${{github.repository }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Download Coverage Artifact
-        uses: actions/download-artifact@v3
-        # if `name` is not specified, all artifacts are downloaded.
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-*
+          merge-multiple: true
 
       - name: Upload Coverage to Coveralls
         uses: coverallsapp/github-action@v2

.github/workflows/docker.yml

@@ -36,31 +36,31 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ${{ github.repository }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           # Checkout the latest release branch
           ref: ${{ github.event.workflow_run.head_sha }}
 
       - name: Set up QEMU (For Linux Arm Containers)
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
 
       # Uses the latest version of Buildx and Buildkit
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           buildkitd-flags: --debug
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker Metadata Information
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         id: docker_metadata
         with:
           github-token: ${{ github.token }}
@@ -74,7 +74,7 @@ jobs:
             type=semver,pattern={{raw}}
 
       - name: Build and Push Docker Image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: Dockerfile

.github/workflows/pypi_publish.yml

@@ -35,7 +35,7 @@ jobs:
     needs: [test, build_wheels_sdist]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: distributions
           path: dist
@@ -57,15 +57,16 @@ jobs:
     if: github.event_name == 'release' && github.event.action == 'published'
     steps:
       - name: Download Artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: distributions
+          pattern: dist-*
+          merge-multiple: true
           path: dist
 
       - name: PYPI Publish
-        uses: pypa/gh-action-pypi-publish@release/v1.6
+        uses: pypa/gh-action-pypi-publish@release/v1.8
         with:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
-          packages_dir: dist/
+          packages-dir: dist/
           verbose: true

.github/workflows/test.yml

@@ -2,6 +2,11 @@ name: Test
 
 on:
   workflow_call:
+    inputs:
+      debug_enabled:
+        type: boolean
+        required: false
+        default: false
 
 permissions:
   contents: read # to fetch code (actions/checkout)
@@ -14,26 +19,28 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.10", "3.11"]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-13, windows-latest]
+    env:
+      GITHUB_WORKSPACE: ${{ github.workspace }}
 
     steps:
       - name: Checkout ${{ github.repository }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Download Example Dataset Cache
-        uses: actions/cache/restore@v3
-        id: dataset-cache
+        uses: actions/cache/restore@v4
         env:
           cache-name: dataset
         with:
           path: ./data/cache/
-          key: huggingface-${{ env.cache-name }}
+          key: huggingface-${{env.cache-name}}
           enableCrossOsArchive: true
+          fail-on-cache-miss: true
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache-dependency-path: "**/pyproject.toml"
@@ -45,11 +52,11 @@ jobs:
 
       - name: Run Tests
         run: |
-          pytest
+          pytest --cov-report=lcov:coverage-${{ join(matrix.*, '-') }}.lcov
 
       - name: Archive Coverage
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-${{ join(matrix.*, '-') }}
           path: |
-            coverage.lcov
+            coverage-${{ join(matrix.*, '-') }}.lcov

conftest.py

@@ -1,5 +1,5 @@
 import os
-import pathlib
+from pathlib import Path
 from typing import Generator, Iterator, Union
 
 import numpy as np
@@ -10,7 +10,7 @@
 def dataset_cache_dir() -> Iterator[Union[str, None]]:
     # Change cache directory if running on CI
     if os.environ.get("CI", None):
-        cache_dir = pathlib.Path("./data/cache/")
+        cache_dir = (Path(os.environ.get("GITHUB_WORKSPACE")) / "data" / "cache").resolve()
     else:
         cache_dir = None
     yield cache_dir

pyproject.toml

@@ -1,17 +1,17 @@
 [build-system]
 requires = [
     "setuptools",
-    "Cython>=0.29,<1",
+    "Cython>=3",
     "numpy>=1.20,<1.24",
-    "setuptools_scm[toml]>=6.2",
+    "setuptools_scm[toml]>=6",
 ]
 build-backend = "setuptools.build_meta"
 
 [project]
 dependencies = [
-    "alpineer==0.1.10",
+    "alpineer==0.1.12",
     "anndata",
-    "Cython>=0.29,<1",
+    "Cython>=3",
     "dask[distributed]",
     "datasets>=2.6,<3.0",
     "dill>=0.3.5,<0.4",
@@ -27,9 +27,9 @@ dependencies = [
     "palettable>=3.3.0,<4",
     "pandas>=2",
     "pillow>=9,<10",
-    "pyFlowSOM>=0.1.15",
+    "pyFlowSOM>=0.1.16",
     "requests>=2.20,<3",
-    "scikit-image<=0.19.3",
+    "scikit-image<0.19.3",
     "scikit-learn>=1.1,<2",
     "graphviz",
     "scipy>=1.7,<2",
@@ -99,6 +99,7 @@ version_scheme = "release-branch-semver"
 local_scheme = "no-local-version"
 
 [tool.cibuildwheel]
+before-all = "uname -a"
 build = ["cp39-*", "cp310-*", "cp311-*"]
 skip = [
     "cp36-*",         # Python 3.6
@@ -115,14 +116,6 @@ skip = [
 
 build-frontend = "build"
 
-# Avoid testing on emulated architectures
-test-skip = [
-    "*-win_arm64",               # Skip testing emulated arm64 biulds on Windows
-    "*-*linux_aarch64",          # Skip testing emulated Linux builds
-    "*-macosx_arm64",            # Skip testing emulated arm64 builds on Intel Macs
-    "*-macosx_universal2:arm64", # Skip testing emulated arm64 portion of universal2 builds
-]
-
 # "manylinux" versioning
 # PEP 600: https://peps.python.org/pep-0600/
 # Build using the manylinux_2_28 image
@@ -144,9 +137,8 @@ repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest
 # Build for Windows x86_64, and ARM 64
 [tool.cibuildwheel.windows]
 archs = ["AMD64", "ARM64"]
-# might not need to repair with delvewheel?
-# before-build = "pip install delvewheel"                          # Use delvewheel on windows
-# repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}"
+before-build = "pip install delvewheel"                          # Use delvewheel on windows
+repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}"
 
 # Coverage
 [tool.coverage.paths]
@@ -177,7 +169,6 @@ addopts = [
     "--randomly-seed=24",
     "--randomly-dont-reorganize",
     "--cov=ark",
-    "--cov-report=lcov",
     "--pycodestyle",
 ]
 console_output_style = "count"