Fix: System testing on DO took too many resources

Problem: Running system testing on DigitalOcean for every push consumed a lot of resources and failed frequently. We now start to have integration testing using `pytest`, which provides a better confidence that things actually work. Solution: Only test on DO open Pull Requests and not every push. In the future, consider only running when merged on `main`.
aleph-im · May 3, 2024 · 0350344 · 0350344
1 parent 8318c43
commit 0350344
Show file tree

Hide file tree

Showing 2 changed files with 113 additions and 95 deletions.
diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml
@@ -1,107 +1,30 @@
-name: "Test on DigitalOcean Droplets"
-
+# These are end-to-end tests running on ephemeral DigitalOcean "Droplet" virtual machines
+# with the different operating systems that are supported.
+#
+# The main focus of these tests is to ensure that the packaging works on all supported platforms
+# and to ensure the compatibility of dependencies (system and vendored) across these platforms.
+name: "Testing on DigitalOcean Droplets"
+
+# Run automatically on main branches, Pull Request updates and allow manual execution using `workflow_dispatch`.
 on:
-  push
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - "opened"
+      - "reopened"
+      - "synchronize"
+      - "ready_for_review"
+  workflow_dispatch:
 
 jobs:
-  tests-python:
-    name: "Test Python code"
-    runs-on: ubuntu-22.04
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
-
-      - name: Install required system packages only for Ubuntu Linux
-        run: |
-          sudo apt-get update
-          sudo apt-get -y upgrade
-          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
-          pip install --upgrade typing-extensions types-PyYAML
-
-      - name: Install required Python packages
-        run: |
-          python3 -m pip install hatch hatch-vcs coverage
-
-      - name: Test style wth ruff, black and isort
-        run: |
-          hatch run lint:style
-
-      - name: Test typing with Mypy
-        run: |
-          hatch run lint:typing
-
-      - name: Install required system packages for installing and running tests
-        run: |
-          sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
-
-      - name: Download and build required files for running tests. Copied from packaging/Makefile.
-        run: |
-          sudo mkdir --parents /opt/firecracker/
-          sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"
-          
-          rm -fr /tmp/firecracker-release
-          mkdir --parents /tmp/firecracker-release /opt/firecracker
-          curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release
-          # Copy binaries:
-          cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker
-          cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer
-          chmod +x /opt/firecracker/firecracker
-          chmod +x /opt/firecracker/jailer
-          
-          find /opt
-
-      - name: "Build custom runtime"
-        run: |
-          sudo apt update
-          sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
-          cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
-
-      - name: "Build example volume"
-        run: |
-          cd examples/volumes && bash build_squashfs.sh
-
-        # Unit tests create and delete network interfaces, and therefore require to run as root
-      - name: Run unit tests
-        run: |
-          sudo python3 -m pip install hatch hatch-vcs coverage
-          sudo hatch run testing:cov
-
-      - name: Upload coverage reports to Codecov
-        uses: codecov/[email protected]
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: aleph-im/aleph-vm
-
-  code-quality-shell:
-    runs-on: ubuntu-22.04
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
-        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
-
-      - name: Install required system packages only for Ubuntu Linux
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y shellcheck
-
-      - name: Run Shellcheck on all shell scripts
-        run: |
-          find ./ -type f -name "*.sh" -exec shellcheck {} \;
-  
 
   run_on_droplet:
     name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}"
     runs-on: ubuntu-latest
     concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias }}"
     timeout-minutes: 10
-    needs:
-      - tests-python
-      - code-quality-shell
 
     strategy:
       matrix:

diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml
@@ -0,0 +1,95 @@
+name: "Test on DigitalOcean Droplets"
+
+on:
+  push
+
+jobs:
+  tests-python:
+    name: "Test Python code"
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install required system packages only for Ubuntu Linux
+        run: |
+          sudo apt-get update
+          sudo apt-get -y upgrade
+          sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables
+          pip install --upgrade typing-extensions types-PyYAML
+
+      - name: Install required Python packages
+        run: |
+          python3 -m pip install hatch hatch-vcs coverage
+
+      - name: Test style wth ruff, black and isort
+        run: |
+          hatch run lint:style
+
+      - name: Test typing with Mypy
+        run: |
+          hatch run lint:typing
+
+      - name: Install required system packages for installing and running tests
+        run: |
+          sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev
+
+      - name: Download and build required files for running tests. Copied from packaging/Makefile.
+        run: |
+          sudo mkdir --parents /opt/firecracker/
+          sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy"
+          
+          rm -fr /tmp/firecracker-release
+          mkdir --parents /tmp/firecracker-release /opt/firecracker
+          curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release
+          # Copy binaries:
+          cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker
+          cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer
+          chmod +x /opt/firecracker/firecracker
+          chmod +x /opt/firecracker/jailer
+          
+          find /opt
+
+      - name: "Build custom runtime"
+        run: |
+          sudo apt update
+          sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86
+          cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../..
+
+      - name: "Build example volume"
+        run: |
+          cd examples/volumes && bash build_squashfs.sh
+
+        # Unit tests create and delete network interfaces, and therefore require to run as root
+      - name: Run unit tests
+        run: |
+          sudo python3 -m pip install hatch hatch-vcs coverage
+          sudo hatch run testing:cov
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: aleph-im/aleph-vm
+
+  code-quality-shell:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Workaround github issue https://github.com/actions/runner-images/issues/7192
+        run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc
+
+      - name: Install required system packages only for Ubuntu Linux
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Run Shellcheck on all shell scripts
+        run: |
+          find ./ -type f -name "*.sh" -exec shellcheck {} \;
+