From 97da7c79e6e239d0ee48f59452f2352e394e7876 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:09:22 -0800 Subject: [PATCH 01/16] Fix hub model test names (#2425) --- tests/hub/test_hub_model.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/hub/test_hub_model.py b/tests/hub/test_hub_model.py index 6029eee21c..eb77ad84fc 100644 --- a/tests/hub/test_hub_model.py +++ b/tests/hub/test_hub_model.py @@ -73,7 +73,7 @@ def prep_scvi_minified_hub_model(save_path: str) -> HubModel: anndata_version=anndata.__version__, data_modalities=["rna"], data_is_annotated=False, - description="scVI model trained on synthetid IID data and uploaded with the minified data.", + description="scVI model trained on synthetic IID data and uploaded with the minified data.", ) return HubModel(model_path, metadata=metadata, model_card=card) @@ -222,25 +222,25 @@ def test_hub_model_create_repo_hf(save_path: str): hub_model = prep_scvi_hub_model(save_path) hub_model.push_to_huggingface_hub( - "scvi-tools/test_scvi_create", + "scvi-tools/test-scvi-create", os.environ["HF_API_TOKEN"], repo_create=True, ) - delete_repo("scvi-tools/test_scvi_create", token=os.environ["HF_API_TOKEN"]) + delete_repo("scvi-tools/test-scvi-create", token=os.environ["HF_API_TOKEN"]) @pytest.mark.private def test_hub_model_push_to_hf(save_path: str): hub_model = prep_scvi_hub_model(save_path) hub_model.push_to_huggingface_hub( - "scvi-tools/test_scvi", + "scvi-tools/test-scvi", os.environ["HF_API_TOKEN"], repo_create=False, ) hub_model = prep_scvi_no_anndata_hub_model(save_path) hub_model.push_to_huggingface_hub( - "scvi-tools/test_scvi_no_anndata", + "scvi-tools/test-scvi-no-anndata", os.environ["HF_API_TOKEN"], repo_create=False, push_anndata=False, @@ -248,7 +248,7 @@ def test_hub_model_push_to_hf(save_path: str): hub_model = prep_scvi_minified_hub_model(save_path) hub_model.push_to_huggingface_hub( - "scvi-tools/test_scvi_minified", + "scvi-tools/test-scvi-minified", os.environ["HF_API_TOKEN"], repo_create=False, ) @@ -256,18 +256,18 @@ def test_hub_model_push_to_hf(save_path: str): @pytest.mark.private def test_hub_model_pull_from_hf(): - hub_model = HubModel.pull_from_huggingface_hub(repo_name="scvi-tools/test_scvi") + hub_model = HubModel.pull_from_huggingface_hub(repo_name="scvi-tools/test-scvi") assert hub_model.model is not None assert hub_model.adata is not None hub_model = HubModel.pull_from_huggingface_hub( - repo_name="scvi-tools/test_scvi_minified" + repo_name="scvi-tools/test-scvi-minified" ) assert hub_model.model is not None assert hub_model.adata is not None hub_model = HubModel.pull_from_huggingface_hub( - repo_name="scvi-tools/test_scvi_no_anndata" + repo_name="scvi-tools/test-scvi-no-anndata" ) with pytest.raises(ValueError): _ = hub_model.model @@ -281,19 +281,19 @@ def test_hub_model_pull_from_hf(): @pytest.mark.private def test_hub_model_push_to_s3(save_path: str): hub_model = prep_scvi_hub_model(save_path) - hub_model.push_to_s3("scvi-tools", "tests/hub/test_scvi") + hub_model.push_to_s3("scvi-tools", "tests/hub/test-scvi") hub_model = prep_scvi_no_anndata_hub_model(save_path) with pytest.raises(ValueError): hub_model.push_to_s3( - "scvi-tools", "tests/hub/test_scvi_no_anndata", push_anndata=True + "scvi-tools", "tests/hub/test-scvi-no-anndata", push_anndata=True ) hub_model.push_to_s3( - "scvi-tools", "tests/hub/test_scvi_no_anndata", push_anndata=False + "scvi-tools", "tests/hub/test-scvi-no-anndata", push_anndata=False ) hub_model = prep_scvi_minified_hub_model(save_path) - hub_model.push_to_s3("scvi-tools", "tests/hub/test_scvi_minified") + hub_model.push_to_s3("scvi-tools", "tests/hub/test-scvi-minified") @pytest.mark.private @@ -302,21 +302,21 @@ def test_hub_model_pull_from_s3(): hub_model = HubModel.pull_from_s3( "scvi-tools", - "tests/hub/test_scvi", + "tests/hub/test-scvi", ) assert hub_model.model is not None assert hub_model.adata is not None - hub_model = HubModel.pull_from_s3("scvi-tools", "tests/hub/test_scvi_minified") + hub_model = HubModel.pull_from_s3("scvi-tools", "tests/hub/test-scvi-minified") assert hub_model.model is not None assert hub_model.adata is not None with pytest.raises(ClientError): - hub_model = HubModel.pull_from_s3("scvi-tools", "tests/hub/test_scvi_no_anndata") + hub_model = HubModel.pull_from_s3("scvi-tools", "tests/hub/test-scvi-no-anndata") hub_model = HubModel.pull_from_s3( "scvi-tools", - "tests/hub/test_scvi_no_anndata", + "tests/hub/test-scvi-no-anndata", pull_anndata=False, ) with pytest.raises(ValueError): From 7e6aaa0371f4e241eea4e81369127b1cb52f7e18 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Thu, 25 Jan 2024 10:37:28 -0800 Subject: [PATCH 02/16] Upgrade to setup-python v5 (#2435) --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test_linux.yml | 2 +- .github/workflows/test_linux_pre.yml | 2 +- .github/workflows/test_linux_private.yml | 2 +- .github/workflows/test_macos.yml | 2 +- .github/workflows/test_windows.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b811f80fff..7d7dc51760 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" cache: "pip" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43e97cc896..b547371983 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: with: filter: blob:none fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" cache: "pip" diff --git a/.github/workflows/test_linux.yml b/.github/workflows/test_linux.yml index 6e32cebf61..cea2298c07 100644 --- a/.github/workflows/test_linux.yml +++ b/.github/workflows/test_linux.yml @@ -34,7 +34,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} cache: "pip" diff --git a/.github/workflows/test_linux_pre.yml b/.github/workflows/test_linux_pre.yml index 36ab3d21ee..d1695dd2c2 100644 --- a/.github/workflows/test_linux_pre.yml +++ b/.github/workflows/test_linux_pre.yml @@ -31,7 +31,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} cache: "pip" diff --git a/.github/workflows/test_linux_private.yml b/.github/workflows/test_linux_private.yml index af636a1610..444841f7aa 100644 --- a/.github/workflows/test_linux_private.yml +++ b/.github/workflows/test_linux_private.yml @@ -44,7 +44,7 @@ jobs: aws-region: us-west-1 role-duration-seconds: 3600 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} cache: "pip" diff --git a/.github/workflows/test_macos.yml b/.github/workflows/test_macos.yml index 02665e40fa..0377a13af7 100644 --- a/.github/workflows/test_macos.yml +++ b/.github/workflows/test_macos.yml @@ -31,7 +31,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} cache: "pip" diff --git a/.github/workflows/test_windows.yml b/.github/workflows/test_windows.yml index 3a1164010d..547fc2363e 100644 --- a/.github/workflows/test_windows.yml +++ b/.github/workflows/test_windows.yml @@ -31,7 +31,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} cache: "pip" From e6e10e828fbd4bfe708afe2d3182401b2fffe72e Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Thu, 25 Jan 2024 11:25:47 -0800 Subject: [PATCH 03/16] Switch cuda docker image tag (#2437) --- .github/workflows/test_linux_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_linux_cuda.yml b/.github/workflows/test_linux_cuda.yml index 11f8a4492a..0e0637c2b0 100644 --- a/.github/workflows/test_linux_cuda.yml +++ b/.github/workflows/test_linux_cuda.yml @@ -26,7 +26,7 @@ jobs: cuda: ["11"] container: - image: martinkim0/scvi-tools:py${{ matrix.python }}-cu${{ matrix.cuda }}-base + image: scverse/scvi-tools:py${{ matrix.python }}-cu${{ matrix.cuda }}-base options: --user root --gpus all environment: From 3190a69c4bf8562d8000d21f62f70d41e2246e26 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:30:42 -0800 Subject: [PATCH 04/16] Add label triage to cuda, private test workflows (#2438) * Add label triage to cuda, private test workflows * Update labels * Fix syntax * Fix syntax * Move check * Fix syntax * Switch labels * Remove private tests from env --- .github/workflows/test_linux_cuda.yml | 9 +++------ .github/workflows/test_linux_private.yml | 9 ++++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_linux_cuda.yml b/.github/workflows/test_linux_cuda.yml index 0e0637c2b0..094d8d0898 100644 --- a/.github/workflows/test_linux_cuda.yml +++ b/.github/workflows/test_linux_cuda.yml @@ -2,8 +2,7 @@ name: Test (Linux, CUDA) on: pull_request: - branches: - [main, 0.15.x, 0.16.x, 0.17.x, 0.18.x, 0.19.x, 0.20.x, 1.0.x, 1.1.x] + branches: [main, 1.0.x, 1.1.x] schedule: - cron: "0 10 * * *" # runs at 10:00 UTC -> 03:00 PST every day workflow_dispatch: @@ -14,6 +13,7 @@ concurrency: jobs: test: + if: (contains(github.event.pull_request.labels.*.name, 'cuda tests') || contains(github.event.pull_request.labels.*.name, 'all tests') || contains(github.event_name, 'schedule') || contains(github.event_name, 'workflow_dispatch')) runs-on: [self-hosted, Linux, X64, CUDA] defaults: run: @@ -29,10 +29,7 @@ jobs: image: scverse/scvi-tools:py${{ matrix.python }}-cu${{ matrix.cuda }}-base options: --user root --gpus all - environment: - name: private_test - - name: ${{ matrix.name }} Python ${{ matrix.python }} CUDA + name: ${{ matrix.name }} Python ${{ matrix.python }} CUDA ${{ matrix.cuda }} env: OS: ${{ matrix.os }} diff --git a/.github/workflows/test_linux_private.yml b/.github/workflows/test_linux_private.yml index 444841f7aa..875cc6db23 100644 --- a/.github/workflows/test_linux_private.yml +++ b/.github/workflows/test_linux_private.yml @@ -2,8 +2,9 @@ name: Test (Linux, private) on: pull_request: - branches: - [main, 0.15.x, 0.16.x, 0.17.x, 0.18.x, 0.19.x, 0.20.x, 1.0.x, 1.1.x] + branches: [main, 1.0.x, 1.1.x] + schedule: + - cron: "0 10 * * *" # runs at 10:00 UTC -> 03:00 PST every day workflow_dispatch: concurrency: @@ -12,6 +13,7 @@ concurrency: jobs: test: + if: (contains(github.event.pull_request.labels.*.name, 'private tests') || contains(github.event.pull_request.labels.*.name, 'all tests') || contains(github.event_name, 'schedule') || contains(github.event_name, 'workflow_dispatch')) runs-on: ${{ matrix.os }} defaults: run: @@ -23,9 +25,6 @@ jobs: os: [ubuntu-latest] python: ["3.11"] - environment: - name: private_test - permissions: id-token: write From ffeee22869b7ac4e0aa34cd92092cb06eaeb6b86 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Thu, 25 Jan 2024 14:38:36 -0800 Subject: [PATCH 05/16] Update workflows (#2441) * Update workflows * Update * Update * Update * Update * Update * Update --- .github/workflows/build.yml | 10 ++++------ .github/workflows/release.yml | 2 +- .github/workflows/test_linux.yml | 8 +++----- .github/workflows/test_linux_cuda.yml | 16 ++++++++++++---- .github/workflows/test_linux_pre.yml | 6 +++--- .github/workflows/test_linux_private.yml | 16 ++++++++++++---- .github/workflows/test_macos.yml | 2 +- .github/workflows/test_windows.yml | 4 ++-- 8 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d7dc51760..86016fef8e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,12 +1,10 @@ -name: Check build +name: Build on: push: - branches: - [main, 0.15.x, 0.16.x, 0.17.x, 0.18.x, 0.19.x, 0.20.x, 1.0.x, 1.1.x] + branches: [main, 1.0.x, 1.1.x] pull_request: - branches: - [main, 0.15.x, 0.16.x, 0.17.x, 0.18.x, 0.19.x, 0.20.x, 1.0.x, 1.1.x] + branches: [main, 1.0.x, 1.1.x] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -20,7 +18,7 @@ jobs: - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.11" cache: "pip" cache-dependency-path: "**/pyproject.toml" - name: Install build dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b547371983..777891a9f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v5 with: - python-version: "3.x" + python-version: "3.11" cache: "pip" - run: pip install build - run: python -m build diff --git a/.github/workflows/test_linux.yml b/.github/workflows/test_linux.yml index cea2298c07..5d3fc69586 100644 --- a/.github/workflows/test_linux.yml +++ b/.github/workflows/test_linux.yml @@ -2,11 +2,9 @@ name: Test (Linux) on: push: - branches: - [main, 0.15.x, 0.16.x, 0.17.x, 0.18.x, 0.19.x, 0.20.x, 1.0.x, 1.1.x] + branches: [main, 1.0.x, 1.1.x] pull_request: - branches: - [main, 0.15.x, 0.16.x, 0.17.x, 0.18.x, 0.19.x, 0.20.x, 1.0.x, 1.1.x] + branches: [main, 1.0.x, 1.1.x] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -25,7 +23,7 @@ jobs: os: [ubuntu-latest] python: ["3.9", "3.10", "3.11"] - name: ${{ matrix.name }} Python ${{ matrix.python }} + name: Integration env: OS: ${{ matrix.os }} diff --git a/.github/workflows/test_linux_cuda.yml b/.github/workflows/test_linux_cuda.yml index 094d8d0898..6589635f13 100644 --- a/.github/workflows/test_linux_cuda.yml +++ b/.github/workflows/test_linux_cuda.yml @@ -1,10 +1,11 @@ -name: Test (Linux, CUDA) +name: Test (Linux) on: pull_request: branches: [main, 1.0.x, 1.1.x] + types: [labeled, synchronize] schedule: - - cron: "0 10 * * *" # runs at 10:00 UTC -> 03:00 PST every day + - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day workflow_dispatch: concurrency: @@ -13,7 +14,14 @@ concurrency: jobs: test: - if: (contains(github.event.pull_request.labels.*.name, 'cuda tests') || contains(github.event.pull_request.labels.*.name, 'all tests') || contains(github.event_name, 'schedule') || contains(github.event_name, 'workflow_dispatch')) + # if PR has label "cuda tests" or "all tests" or if scheduled or manually triggered + if: >- + ( + contains(github.event.pull_request.labels.*.name, 'cuda tests') || + contains(github.event.pull_request.labels.*.name, 'all tests') || + contains(github.event_name, 'schedule') || + contains(github.event_name, 'workflow_dispatch') + ) runs-on: [self-hosted, Linux, X64, CUDA] defaults: run: @@ -29,7 +37,7 @@ jobs: image: scverse/scvi-tools:py${{ matrix.python }}-cu${{ matrix.cuda }}-base options: --user root --gpus all - name: ${{ matrix.name }} Python ${{ matrix.python }} CUDA ${{ matrix.cuda }} + name: CUDA env: OS: ${{ matrix.os }} diff --git a/.github/workflows/test_linux_pre.yml b/.github/workflows/test_linux_pre.yml index d1695dd2c2..5e33248d40 100644 --- a/.github/workflows/test_linux_pre.yml +++ b/.github/workflows/test_linux_pre.yml @@ -1,8 +1,8 @@ -name: Test (Linux, prerelease) +name: Test (Linux, prereleases) on: schedule: - - cron: "0 10 * * *" # runs at 10:00 UTC -> 03:00 PST every day + - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day workflow_dispatch: concurrency: @@ -22,7 +22,7 @@ jobs: os: [ubuntu-latest] python: ["3.9", "3.10", "3.11", "3.12"] - name: ${{ matrix.name }} Python ${{ matrix.python }} prereleases + name: Prereleases env: OS: ${{ matrix.os }} diff --git a/.github/workflows/test_linux_private.yml b/.github/workflows/test_linux_private.yml index 875cc6db23..c2bc2a3396 100644 --- a/.github/workflows/test_linux_private.yml +++ b/.github/workflows/test_linux_private.yml @@ -1,10 +1,11 @@ -name: Test (Linux, private) +name: Test (Linux) on: pull_request: branches: [main, 1.0.x, 1.1.x] + types: [labeled, synchronize] schedule: - - cron: "0 10 * * *" # runs at 10:00 UTC -> 03:00 PST every day + - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day workflow_dispatch: concurrency: @@ -13,7 +14,14 @@ concurrency: jobs: test: - if: (contains(github.event.pull_request.labels.*.name, 'private tests') || contains(github.event.pull_request.labels.*.name, 'all tests') || contains(github.event_name, 'schedule') || contains(github.event_name, 'workflow_dispatch')) + # if PR has label "private tests" or "all tests" or if scheduled or manually triggered + if: >- + ( + contains(github.event.pull_request.labels.*.name, 'private tests') || + contains(github.event.pull_request.labels.*.name, 'all tests') || + contains(github.event_name, 'schedule') || + contains(github.event_name, 'workflow_dispatch') + ) runs-on: ${{ matrix.os }} defaults: run: @@ -28,7 +36,7 @@ jobs: permissions: id-token: write - name: ${{ matrix.name }} Python ${{ matrix.python }} private + name: Private env: OS: ${{ matrix.os }} diff --git a/.github/workflows/test_macos.yml b/.github/workflows/test_macos.yml index 0377a13af7..e7c48c4836 100644 --- a/.github/workflows/test_macos.yml +++ b/.github/workflows/test_macos.yml @@ -22,7 +22,7 @@ jobs: os: [macos-latest] python: ["3.9", "3.10", "3.11"] - name: ${{ matrix.name }} Python ${{ matrix.python }} + name: Integration env: OS: ${{ matrix.os }} diff --git a/.github/workflows/test_windows.yml b/.github/workflows/test_windows.yml index 547fc2363e..144411d8ef 100644 --- a/.github/workflows/test_windows.yml +++ b/.github/workflows/test_windows.yml @@ -2,7 +2,7 @@ name: Test (Windows) on: schedule: - - cron: "0 10 * * *" # runs at 10:00 UTC -> 03:00 PST every day + - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day workflow_dispatch: concurrency: @@ -22,7 +22,7 @@ jobs: os: [windows-latest] python: ["3.9", "3.10", "3.11"] - name: ${{ matrix.name }} Python ${{ matrix.python }} + name: Integration env: OS: ${{ matrix.os }} From 622bf4740d15219374ad76bac22ec8ba80bacc13 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Thu, 25 Jan 2024 15:05:31 -0800 Subject: [PATCH 06/16] Update workflow for pushes (#2443) --- .github/workflows/build.yml | 4 ++-- .github/workflows/test_linux.yml | 4 ++-- .github/workflows/test_linux_cuda.yml | 4 ++-- .github/workflows/test_linux_private.yml | 11 +++++++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 86016fef8e..0c329ebf84 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,9 +2,9 @@ name: Build on: push: - branches: [main, 1.0.x, 1.1.x] + branches: [main, 1.1.x] pull_request: - branches: [main, 1.0.x, 1.1.x] + branches: [main, 1.1.x] concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/test_linux.yml b/.github/workflows/test_linux.yml index 5d3fc69586..a9efb39b20 100644 --- a/.github/workflows/test_linux.yml +++ b/.github/workflows/test_linux.yml @@ -2,9 +2,9 @@ name: Test (Linux) on: push: - branches: [main, 1.0.x, 1.1.x] + branches: [main, 1.1.x] pull_request: - branches: [main, 1.0.x, 1.1.x] + branches: [main, 1.1.x] concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/test_linux_cuda.yml b/.github/workflows/test_linux_cuda.yml index 6589635f13..66a5be9287 100644 --- a/.github/workflows/test_linux_cuda.yml +++ b/.github/workflows/test_linux_cuda.yml @@ -2,8 +2,8 @@ name: Test (Linux) on: pull_request: - branches: [main, 1.0.x, 1.1.x] - types: [labeled, synchronize] + branches: [main, 1.1.x] + types: [labeled, synchronize, opened] schedule: - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day workflow_dispatch: diff --git a/.github/workflows/test_linux_private.yml b/.github/workflows/test_linux_private.yml index c2bc2a3396..4c350290a9 100644 --- a/.github/workflows/test_linux_private.yml +++ b/.github/workflows/test_linux_private.yml @@ -2,8 +2,10 @@ name: Test (Linux) on: pull_request: - branches: [main, 1.0.x, 1.1.x] - types: [labeled, synchronize] + branches: [main, 1.1.x] + types: [labeled, synchronize, opened] + push: + branches: [main, 1.1.x] schedule: - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day workflow_dispatch: @@ -14,13 +16,14 @@ concurrency: jobs: test: - # if PR has label "private tests" or "all tests" or if scheduled or manually triggered + # if PR has label "private tests" or "all tests" or if scheduled or manually triggered or on push if: >- ( contains(github.event.pull_request.labels.*.name, 'private tests') || contains(github.event.pull_request.labels.*.name, 'all tests') || contains(github.event_name, 'schedule') || - contains(github.event_name, 'workflow_dispatch') + contains(github.event_name, 'workflow_dispatch') || + contains(github.event_name, 'push') ) runs-on: ${{ matrix.os }} defaults: From 2c63d44e7218747c20d60a1acfcf8473482869d5 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Thu, 25 Jan 2024 15:23:31 -0800 Subject: [PATCH 07/16] Change workflow names (#2444) * Change workflow names * Update --- .github/workflows/test_linux_cuda.yml | 4 ++-- .github/workflows/test_linux_pre.yml | 13 ++++++++++++- .github/workflows/test_linux_private.yml | 4 ++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_linux_cuda.yml b/.github/workflows/test_linux_cuda.yml index 66a5be9287..954059ef42 100644 --- a/.github/workflows/test_linux_cuda.yml +++ b/.github/workflows/test_linux_cuda.yml @@ -1,4 +1,4 @@ -name: Test (Linux) +name: Test (Linux, CUDA) on: pull_request: @@ -37,7 +37,7 @@ jobs: image: scverse/scvi-tools:py${{ matrix.python }}-cu${{ matrix.cuda }}-base options: --user root --gpus all - name: CUDA + name: Integration (CUDA) env: OS: ${{ matrix.os }} diff --git a/.github/workflows/test_linux_pre.yml b/.github/workflows/test_linux_pre.yml index 5e33248d40..69d944a72c 100644 --- a/.github/workflows/test_linux_pre.yml +++ b/.github/workflows/test_linux_pre.yml @@ -1,6 +1,9 @@ name: Test (Linux, prereleases) on: + pull_request: + branches: [main, 1.1.x] + types: [labeled, synchronize, opened] schedule: - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day workflow_dispatch: @@ -11,6 +14,14 @@ concurrency: jobs: test: + # if PR has label "cuda tests" or "all tests" or if scheduled or manually triggered + if: >- + ( + contains(github.event.pull_request.labels.*.name, 'prerelease tests') || + contains(github.event.pull_request.labels.*.name, 'all tests') || + contains(github.event_name, 'schedule') || + contains(github.event_name, 'workflow_dispatch') + ) runs-on: ${{ matrix.os }} defaults: run: @@ -22,7 +33,7 @@ jobs: os: [ubuntu-latest] python: ["3.9", "3.10", "3.11", "3.12"] - name: Prereleases + name: Integration (Prereleases) env: OS: ${{ matrix.os }} diff --git a/.github/workflows/test_linux_private.yml b/.github/workflows/test_linux_private.yml index 4c350290a9..35d598d5ad 100644 --- a/.github/workflows/test_linux_private.yml +++ b/.github/workflows/test_linux_private.yml @@ -1,4 +1,4 @@ -name: Test (Linux) +name: Test (Linux, private) on: pull_request: @@ -39,7 +39,7 @@ jobs: permissions: id-token: write - name: Private + name: Unit (Private) env: OS: ${{ matrix.os }} From 9b5f5537ff5f8f1066e6a6170c4d5e6660b56793 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Mon, 29 Jan 2024 09:38:37 -0800 Subject: [PATCH 08/16] Fix reinitialization error in GIMVI (#2446) Fixes #2273. Error occurs when re-initializing GIMVI with the same spatial dataset due to the following code in `GIMVI.__init__` that in-place modified an `.obs` column: ``` adata_seq_n_batches = sum_stats[0]["n_batch"] adata_spatial.obs[ self.adata_managers["spatial"] .data_registry[REGISTRY_KEYS.BATCH_KEY] .attr_key ] += adata_seq_n_batches ``` This was necessary as the spatial batches and GEX batches were 0-indexed, but we wanted to distinguish between them. The fix checks whether the obs column's minimum is 0, and if so, we add the offset. Otherwise, we don't, since that means the column was updated before within the same session. --- docs/release_notes/index.md | 1 + scvi/external/gimvi/_model.py | 15 +++++++++++---- tests/external/gimvi/test_gimvi.py | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/release_notes/index.md b/docs/release_notes/index.md index 313b131943..26eb6314ac 100644 --- a/docs/release_notes/index.md +++ b/docs/release_notes/index.md @@ -76,6 +76,7 @@ is available in the [commit logs](https://github.com/scverse/scvi-tools/commits/ - Fix initialization of {class}`scvi.distributions.NegativeBinomial` and {class}`scvi.distributions.ZeroInflatedNegativeBinomial` when `validate_args=True` and optional parameters not passed in {pr}`2395`. +- Fix error when re-initializing {class}`scvi.external.GIMVI` with the same datasets {pr}`2446`. #### Changed diff --git a/scvi/external/gimvi/_model.py b/scvi/external/gimvi/_model.py index b7a1a202b1..15c8ad6b9c 100644 --- a/scvi/external/gimvi/_model.py +++ b/scvi/external/gimvi/_model.py @@ -119,14 +119,21 @@ def __init__( total_genes = n_inputs[0] - # since we are combining datasets, we need to increment the batch_idx - # of one of the datasets adata_seq_n_batches = sum_stats[0]["n_batch"] - adata_spatial.obs[ + adata_spatial_batch = adata_spatial.obs[ self.adata_managers["spatial"] .data_registry[REGISTRY_KEYS.BATCH_KEY] .attr_key - ] += adata_seq_n_batches + ] + if np.min(adata_spatial_batch) == 0: + # see #2446 + # since we are combining datasets, we need to increment the batch_idx of one of the + # datasets. we only need to do this once so we check if the min is 0 + adata_spatial.obs[ + self.adata_managers["spatial"] + .data_registry[REGISTRY_KEYS.BATCH_KEY] + .attr_key + ] += adata_seq_n_batches n_batches = sum(s["n_batch"] for s in sum_stats) diff --git a/tests/external/gimvi/test_gimvi.py b/tests/external/gimvi/test_gimvi.py index 79e35e9908..956ce7b349 100644 --- a/tests/external/gimvi/test_gimvi.py +++ b/tests/external/gimvi/test_gimvi.py @@ -169,3 +169,23 @@ def test_gimvi_model_library_size(): model.train(1, check_val_every_n_epoch=1, train_size=0.5) model.get_latent_representation() model.get_imputed_values() + + +def test_gimvi_reinit(): + # see #2446 + adata_seq = synthetic_iid() + adata_spatial = synthetic_iid() + GIMVI.setup_anndata( + adata_seq, + batch_key="batch", + labels_key="labels", + ) + GIMVI.setup_anndata( + adata_spatial, + batch_key="batch", + labels_key="labels", + ) + model = GIMVI(adata_seq, adata_spatial) + model.train(max_epochs=1) + model = GIMVI(adata_seq, adata_spatial) + model.train(max_epochs=1) From 9b99f95be35120046c14ca69bd44f10c2d2188ee Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:15:12 -0800 Subject: [PATCH 09/16] Remove black and flake8 as deps (#2450) --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 79513c6ea7..7ce2ca3e0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,8 +66,6 @@ tests = [ "scvi-tools[optional]" ] # dependencies for running the test suite editing = [ - "black", - "flake8", "jupyter", "nbformat", "pre-commit", From fec75038acce822cbddcbc7b37b96ac4e0ac843c Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:03:56 -0800 Subject: [PATCH 10/16] Update tutorials head (#2453) --- docs/tutorials/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/notebooks b/docs/tutorials/notebooks index b7dfdfba51..ca21ccffd0 160000 --- a/docs/tutorials/notebooks +++ b/docs/tutorials/notebooks @@ -1 +1 @@ -Subproject commit b7dfdfba519a8d821a04b2bee68579c0c003bbf4 +Subproject commit ca21ccffd0a0bd01d2649faedd346660b8e4c37d From 655c79f03be5075330be6597351ae5821337d2d5 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:06:59 -0800 Subject: [PATCH 11/16] Add macos m1 test workflow (#2455) --- .github/workflows/test_macos_m1.yml | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/test_macos_m1.yml diff --git a/.github/workflows/test_macos_m1.yml b/.github/workflows/test_macos_m1.yml new file mode 100644 index 0000000000..5b80ef267f --- /dev/null +++ b/.github/workflows/test_macos_m1.yml @@ -0,0 +1,59 @@ +name: Test (MacOS M1) + +on: + schedule: + - cron: "0 10 * * *" # runs at 10:00 UTC -> 03:00 PST every day + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -e {0} # -e to fail on error + + strategy: + fail-fast: false + matrix: + os: [macos-14] + python: ["3.9", "3.10", "3.11"] + + name: Integration + + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python }} + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" + + - name: Install test dependencies + run: | + python -m pip install --upgrade pip wheel + + - name: Install dependencies + run: | + pip install ".[tests]" + + - name: Test + env: + MPLBACKEND: agg + PLATFORM: ${{ matrix.os }} + DISPLAY: :42 + run: | + coverage run -m pytest -v --color=yes + - name: Report coverage + run: | + coverage report + - name: Upload coverage + uses: codecov/codecov-action@v3 From e6f8e3257f18113d2c6e77dc6afb59350068a88b Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:50:35 -0800 Subject: [PATCH 12/16] Remove unused files (#2457) --- .gitattributes | 2 -- setup.py | 8 -------- 2 files changed, 10 deletions(-) delete mode 100644 .gitattributes delete mode 100644 setup.py diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index b6115e6e49..0000000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -*.py linguist-language=python -*.ipynb linguist-documentation diff --git a/setup.py b/setup.py deleted file mode 100644 index 19c8a02a9b..0000000000 --- a/setup.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python - -# This is a shim to hopefully allow Github to detect the package, build is done with poetry - -import setuptools - -if __name__ == "__main__": - setuptools.setup(name="scvi-tools") From e557be476208cdeda31129814d5973c5e5143a3b Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:53:53 -0800 Subject: [PATCH 13/16] Update codecov and readthedocs yaml (#2458) --- codecov.yml | 14 +++++++++----- readthedocs.yml | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/codecov.yml b/codecov.yml index 3eeed4ba11..0e2efd00c7 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,12 +1,16 @@ -# Run to check if valid -# curl --data-binary @codecov.yml https://codecov.io/validate +codecov: + require_ci_to_pass: no + coverage: status: project: default: target: 85% threshold: 1% - patch: off + patch: false + changes: false -ignore: - - "scvi/_compat.py" +comment: + layout: diff, flags, files + behavior: once + require_base: no diff --git a/readthedocs.yml b/readthedocs.yml index dd3482d233..acedf0a788 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -2,7 +2,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.10" + python: "3.11" sphinx: configuration: docs/conf.py python: From dfd14a2e6af2a069d1b9364030a9e22f002353b0 Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Thu, 1 Feb 2024 10:54:59 -0800 Subject: [PATCH 14/16] Bump version to 1.1.0 (#2462) --- docs/release_notes/index.md | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/release_notes/index.md b/docs/release_notes/index.md index 26eb6314ac..ed4fe0bfa7 100644 --- a/docs/release_notes/index.md +++ b/docs/release_notes/index.md @@ -7,7 +7,7 @@ is available in the [commit logs](https://github.com/scverse/scvi-tools/commits/ [keep a changelog]: https://keepachangelog.com/en/1.0.0/ [semantic versioning]: https://semver.org/spec/v2.0.0.html -### 1.1.0 (2023-12-DD) +### 1.1.0 (2024-02-DD) #### Added @@ -111,7 +111,7 @@ is available in the [commit logs](https://github.com/scverse/scvi-tools/commits/ ## Version 1.0 -### 1.0.4 (2023-10-xx) +### 1.0.4 (2023-10-13) ### Added diff --git a/pyproject.toml b/pyproject.toml index 7ce2ca3e0c..6906a3fdf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = ["hatchling"] [project] name = "scvi-tools" -version = "1.1.0rc2" +version = "1.1.0" description = "Deep probabilistic analysis of single-cell omics data." readme = "README.md" requires-python = ">=3.9" From 3c020c7bec655554fbc13cbf31ad9bff4670640c Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Thu, 1 Feb 2024 10:55:43 -0800 Subject: [PATCH 15/16] Add scANVI fix tutorial to index (#2463) Add scANVI fix tutorail to index --- docs/tutorials/index_scrna.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tutorials/index_scrna.md b/docs/tutorials/index_scrna.md index 2fab9bcb54..507145a564 100644 --- a/docs/tutorials/index_scrna.md +++ b/docs/tutorials/index_scrna.md @@ -15,4 +15,5 @@ notebooks/scrna/cellassign_tutorial notebooks/scrna/amortized_lda notebooks/scrna/scVI_DE_worm notebooks/scrna/contrastiveVI_tutorial +notebooks/scrna/scanvi_fix ``` From 0f560d5102468d2389e3b4844fd8d63633d38adb Mon Sep 17 00:00:00 2001 From: Martin Kim <46072231+martinkim0@users.noreply.github.com> Date: Fri, 2 Feb 2024 09:25:55 -0800 Subject: [PATCH 16/16] Update tutorials head (#2466) * Add scANVI fix tutorail to index * Update tutorials head --- docs/tutorials/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/notebooks b/docs/tutorials/notebooks index ca21ccffd0..e773cc9f9f 160000 --- a/docs/tutorials/notebooks +++ b/docs/tutorials/notebooks @@ -1 +1 @@ -Subproject commit ca21ccffd0a0bd01d2649faedd346660b8e4c37d +Subproject commit e773cc9f9f9d116c0b5a8a8c813c4d890743ca5c