diff --git a/.github/workflows/test-on-droplets-matrix.yml b/.github/workflows/test-on-droplets-matrix.yml index c67c1688f..847469387 100644 --- a/.github/workflows/test-on-droplets-matrix.yml +++ b/.github/workflows/test-on-droplets-matrix.yml @@ -1,107 +1,30 @@ -name: "Test on DigitalOcean Droplets" - +# These are end-to-end tests running on ephemeral DigitalOcean "Droplet" virtual machines +# with the different operating systems that are supported. +# +# The main focus of these tests is to ensure that the packaging works on all supported platforms +# and to ensure the compatibility of dependencies (system and vendored) across these platforms. +name: "Testing on DigitalOcean Droplets" + +# Run automatically on main branches, Pull Request updates and allow manual execution using `workflow_dispatch`. on: - push + push: + branches: + - main + pull_request: + types: + - "opened" + - "reopened" + - "synchronize" + - "ready_for_review" + workflow_dispatch: jobs: - tests-python: - name: "Test Python code" - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v4 - - - name: Workaround github issue https://github.com/actions/runner-images/issues/7192 - run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc - - - name: Install required system packages only for Ubuntu Linux - run: | - sudo apt-get update - sudo apt-get -y upgrade - sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables - pip install --upgrade typing-extensions types-PyYAML - - - name: Install required Python packages - run: | - python3 -m pip install hatch hatch-vcs coverage - - - name: Test style wth ruff, black and isort - run: | - hatch run lint:style - - - name: Test typing with Mypy - run: | - hatch run lint:typing - - - name: Install required system packages for installing and running tests - run: | - sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev - - - name: Download and build required files for running tests. Copied from packaging/Makefile. - run: | - sudo mkdir --parents /opt/firecracker/ - sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy" - - rm -fr /tmp/firecracker-release - mkdir --parents /tmp/firecracker-release /opt/firecracker - curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release - # Copy binaries: - cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker - cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer - chmod +x /opt/firecracker/firecracker - chmod +x /opt/firecracker/jailer - - find /opt - - - name: "Build custom runtime" - run: | - sudo apt update - sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86 - cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../.. - - - name: "Build example volume" - run: | - cd examples/volumes && bash build_squashfs.sh - - # Unit tests create and delete network interfaces, and therefore require to run as root - - name: Run unit tests - run: | - sudo python3 -m pip install hatch hatch-vcs coverage - sudo hatch run testing:cov - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4.0.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: aleph-im/aleph-vm - - code-quality-shell: - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v4 - - - name: Workaround github issue https://github.com/actions/runner-images/issues/7192 - run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc - - - name: Install required system packages only for Ubuntu Linux - run: | - sudo apt-get update - sudo apt-get install -y shellcheck - - - name: Run Shellcheck on all shell scripts - run: | - find ./ -type f -name "*.sh" -exec shellcheck {} \; - run_on_droplet: name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}" runs-on: ubuntu-latest concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias }}" timeout-minutes: 10 - needs: - - tests-python - - code-quality-shell strategy: matrix: diff --git a/.github/workflows/test-using-pytest.yml b/.github/workflows/test-using-pytest.yml new file mode 100644 index 000000000..ba77cea22 --- /dev/null +++ b/.github/workflows/test-using-pytest.yml @@ -0,0 +1,95 @@ +name: "Test on DigitalOcean Droplets" + +on: + push + +jobs: + tests-python: + name: "Test Python code" + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Workaround github issue https://github.com/actions/runner-images/issues/7192 + run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc + + - name: Install required system packages only for Ubuntu Linux + run: | + sudo apt-get update + sudo apt-get -y upgrade + sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables + pip install --upgrade typing-extensions types-PyYAML + + - name: Install required Python packages + run: | + python3 -m pip install hatch hatch-vcs coverage + + - name: Test style wth ruff, black and isort + run: | + hatch run lint:style + + - name: Test typing with Mypy + run: | + hatch run lint:typing + + - name: Install required system packages for installing and running tests + run: | + sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev + + - name: Download and build required files for running tests. Copied from packaging/Makefile. + run: | + sudo mkdir --parents /opt/firecracker/ + sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy" + + rm -fr /tmp/firecracker-release + mkdir --parents /tmp/firecracker-release /opt/firecracker + curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release + # Copy binaries: + cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker + cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer + chmod +x /opt/firecracker/firecracker + chmod +x /opt/firecracker/jailer + + find /opt + + - name: "Build custom runtime" + run: | + sudo apt update + sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86 + cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../.. + + - name: "Build example volume" + run: | + cd examples/volumes && bash build_squashfs.sh + + # Unit tests create and delete network interfaces, and therefore require to run as root + - name: Run unit tests + run: | + sudo python3 -m pip install hatch hatch-vcs coverage + sudo hatch run testing:cov + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: aleph-im/aleph-vm + + code-quality-shell: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Workaround github issue https://github.com/actions/runner-images/issues/7192 + run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc + + - name: Install required system packages only for Ubuntu Linux + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + + - name: Run Shellcheck on all shell scripts + run: | + find ./ -type f -name "*.sh" -exec shellcheck {} \; +