ci: tweak the Python workflows

Remove manual installations of PyTorch everywhere, as the latter should be handled while installing the library through `pip`. Remove `-e` flag while installing our packages, and refer to them with `./`. Remove `s3torchbenchmarking` unit tests runs, unnecessary for now. Reformat some workflow files (cosmetic). Use `pip` instead of `python -m pip`. Drop support (in worfklows) for macOS x86_64 architecture, since PyTorch dropped support to since v2.3 (https://dev-discuss.pytorch.org/t/pytorch-macos-x86-builds-deprecation-starting-january-2024/1690), and future DCP implementations will require that version.
awslabs · Oct 24, 2024 · 4d5b08b · 4d5b08b
1 parent 9a0c89f
commit 4d5b08b
Show file tree

Hide file tree

Showing 6 changed files with 56 additions and 70 deletions.
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -11,29 +11,26 @@ jobs:
   docs:
     runs-on: ubuntu-22.04
     steps:
-        - name: Checkout code
-          uses: actions/checkout@v4
-
-        - name: Setup Python
-          uses: actions/setup-python@v5
-
-        - name: Install local packages to update versions
-          run: |
-            python -m pip install --upgrade pip
-            # Manually install CPU-only version of torch so we're not carrying around giant GPU drivers/kernels
-            python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
-            python -m pip install -e "s3torchconnectorclient"
-            python -m pip install -e "s3torchconnector"
-
-        - name: Install dependencies
-          run: |
-            pip install sphinx sphinx_rtd_theme sphinx-autoapi ghp-import
-
-        - name: Sphinx build
-          run: |
-            cd s3torchconnector/docs
-            sphinx-build -b html . _build/html
-
-        - name: Import docs
-          run: |
-            ghp-import -n -p -f s3torchconnector/docs/_build/html
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+
+      - name: Install local packages to update versions
+        run: |
+          pip install ./s3torchconnectorclient
+          pip install ./s3torchconnector
+
+      - name: Install dependencies
+        run: |
+          pip install sphinx sphinx_rtd_theme sphinx-autoapi ghp-import
+
+      - name: Sphinx build
+        run: |
+          cd s3torchconnector/docs
+          sphinx-build -b html . _build/html
+
+      - name: Import docs
+        run: |
+          ghp-import -n -p -f s3torchconnector/docs/_build/html
diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
 
     steps:
       - name: Checkout code
@@ -42,24 +42,18 @@ jobs:
           cache: 'pip'
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          # Manually install CPU-only version of torch so we're not carrying around giant GPU drivers/kernels
-          python -m pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu
-          python -m pip install -e "s3torchconnectorclient[test]"
-          python -m pip install -e "s3torchconnector[test]"
-          python -m pip install -e "s3torchbenchmarking[test]"
+          pip install "./s3torchconnectorclient[test]"
+          pip install "./s3torchconnector[test]"
 
-      - name: s3torchconnectorclient unit tests
+      - name: Run s3torchconnectorclient unit tests
         run: pytest s3torchconnectorclient/python/tst/unit --hypothesis-profile ci --hypothesis-show-statistics -c ./
-      - name: s3torchconnector unit tests
+      - name: Run s3torchconnector unit tests
         run: pytest s3torchconnector/tst/unit --ignore s3torchconnector/tst/unit/lightning --hypothesis-profile ci --hypothesis-show-statistics -c ./
-      - name: s3torchbenchmarking unit tests
-        run: pytest s3torchbenchmarking/tst --hypothesis-profile ci --hypothesis-show-statistics -c ./
 
       - name: Install Lightning dependency
         run: |
-          python -m pip install -e "s3torchconnector[lightning-tests]"
-      - name: s3torchconnector lightning unit tests
+          pip install "./s3torchconnector[lightning-tests]"
+      - name: Run s3torchconnector Lightning unit tests
         run: pytest s3torchconnector/tst/unit/lightning --hypothesis-profile ci --hypothesis-show-statistics -c ./
 
   lint:
@@ -85,11 +79,9 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install flake8 black mypy
-          python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
-          python -m pip install ./s3torchconnectorclient
-          python -m pip install ./s3torchconnector[lightning]
+          pip install flake8 black mypy
+          pip install "./s3torchconnectorclient"
+          pip install "./s3torchconnector[lightning]"
 
       - name: Lint with flake8
         run: |
@@ -116,8 +108,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install pipreqs safety
+          pip install pipreqs safety
 
       - name: Generate requirements
         run: |

diff --git a/.github/workflows/python-integration.yml b/.github/workflows/python-integration.yml
@@ -24,8 +24,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runner: [ubuntu-22.04, macos-13]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        runner: [ ubuntu-22.04, macos-13-arm64 ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
         test-run:
           - name: "S3"
             bucket: ${{ vars.S3_BUCKET }}
@@ -75,13 +75,10 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          # Manually install CPU-only version of torch so we're not carrying around giant GPU drivers/kernels
-          python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
-          python -m pip install -e "s3torchconnectorclient[test,e2e]"
-          python -m pip install -e "s3torchconnector[test,e2e]"
+          pip install "./s3torchconnectorclient[test]"
+          pip install "./s3torchconnector[test,e2e]"
 
-      - name: s3torchconnector ${{ matrix.test-run.name }} integration tests
+      - name: Run s3torchconnector ${{ matrix.test-run.name }} integration tests
         run: |
           CI_REGION=${{ matrix.test-run.region }} \
           CI_BUCKET=${{ matrix.test-run.bucket }} \
@@ -90,15 +87,15 @@ jobs:
 
       - name: Install Lightning dependency
         run: |
-          python -m pip install -e "s3torchconnector[lightning-tests]"
+          pip install "./s3torchconnector[lightning-tests]"
       - name: s3torchconnector lightning e2e tests
         run: |
           CI_REGION=${{ matrix.test-run.region }} \
           CI_BUCKET=${{ matrix.test-run.bucket }} \
           CI_STORAGE_CLASS=${{ matrix.test-run.storage-class }} \
           pytest s3torchconnector/tst/e2e/test_e2e_s3_lightning_checkpoint.py -n auto
 
-      - name: s3torchconnectorclient ${{ matrix.test-run.name }} integration tests
+      - name: Run s3torchconnectorclient ${{ matrix.test-run.name }} integration tests
         run: |
           CI_REGION=${{ matrix.test-run.region }} \
           CI_BUCKET=${{ matrix.test-run.bucket }} \

diff --git a/.github/workflows/rust-checks.yml b/.github/workflows/rust-checks.yml
@@ -16,12 +16,12 @@ jobs:
     strategy:
       matrix:
         checks:
-        # The advisories check is used to detect issues for crates by looking in an advisory database.
+          # The advisories check is used to detect issues for crates by looking in an advisory database.
           - advisories
-        # The bans check is used to deny (or allow) specific crates, as well as detect and handle multiple
-        # versions of the same crate.
-        # The licenses check is used to verify that every crate you use has license terms you find acceptable.
-        # The sources check ensures crates only come from sources you trust.
+          # The bans check is used to deny (or allow) specific crates, as well as detect and handle multiple
+          # versions of the same crate.
+          # The licenses check is used to verify that every crate you use has license terms you find acceptable.
+          # The sources check ensures crates only come from sources you trust.
           - bans licenses sources
     steps:
       - name: Checkout code
@@ -64,7 +64,7 @@ jobs:
     name: Rust tests
     strategy:
       matrix:
-        runner: [ubuntu-22.04, macos-13]
+        runner: [ ubuntu-22.04, macos-13-arm64 ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -37,13 +37,13 @@ jobs:
           - runner: codebuild-${{ vars.CODEBUILD_PROJECT_NAME }}-${{ github.run_id }}-${{ github.run_attempt }}-arm-3.0-large
             kind: manylinux
             arch: aarch64
-#          - runner: ubuntu-20.04
-#            kind: manylinux
-#            arch: x86_64
-          - runner: macos-13
+          #          - runner: ubuntu-20.04
+          #            kind: manylinux
+          #            arch: x86_64
+          - runner: macos-13-arm64
             kind: macosx
             arch: x86_64
-          - runner: macos-14
+          - runner: macos-14-arm64
             kind: macosx
             arch: arm64
     permissions:
@@ -151,5 +151,5 @@ jobs:
       - name: Merge Artifacts
         uses: actions/upload-artifact/merge@v4
         with:
-           name: wheels
-           pattern: wheels-*
+          name: wheels
+          pattern: wheels-*
diff --git a/s3torchconnectorclient/pyproject.toml b/s3torchconnectorclient/pyproject.toml
@@ -32,7 +32,8 @@ test = [
     "flake8",
     "black",
     "mypy",
-    "Pillow"
+    "numpy",
+    "Pillow",
 ]
 
 [tool.setuptools.packages]