typecheck hack #737
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Test on DigitalOcean Droplets" | |
| on: | |
| push | |
| jobs: | |
| tests-python: | |
| name: "Test Python code" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Workaround github issue https://github.com/actions/runner-images/issues/7192 | |
| run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc | |
| - name: Install required system packages only for Ubuntu Linux | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y upgrade | |
| sudo apt-get install -y python3 python3-pip python3-aiohttp python3-msgpack python3-aiodns python3-alembic python3-sqlalchemy python3-setproctitle redis python3-aioredis python3-psutil sudo acl curl systemd-container squashfs-tools debootstrap python3-packaging python3-cpuinfo python3-nftables python3-jsonschema nftables | |
| pip install --upgrade typing-extensions types-PyYAML | |
| - name: Install required Python packages | |
| run: | | |
| python3 -m pip install hatch hatch-vcs coverage | |
| - name: Test style wth ruff, black and isort | |
| run: | | |
| hatch run lint:style | |
| - name: Test typing with Mypy | |
| run: | | |
| hatch run lint:typing | |
| - name: Install required system packages for installing and running tests | |
| run: | | |
| sudo apt-get install libsystemd-dev cmake libdbus-1-dev libglib2.0-dev | |
| - name: Download and build required files for running tests | |
| run: | | |
| sudo mkdir --parents /opt/firecracker/ | |
| sudo curl -fsSL -o "/opt/firecracker/vmlinux.bin" "https://ipfs.aleph.cloud/ipfs/bafybeiaj2lf6g573jiulzacvkyw4zzav7dwbo5qbeiohoduopwxs2c6vvy" | |
| rm -fr /tmp/firecracker-release | |
| mkdir --parents /tmp/firecracker-release /opt/firecracker | |
| curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v1.5.0/firecracker-v1.5.0-x86_64.tgz | tar -xz --no-same-owner --directory /tmp/firecracker-release | |
| # Copy binaries: | |
| cp /tmp/firecracker-release/release-v*/firecracker-v*[!.debug] /opt/firecracker/firecracker | |
| cp /tmp/firecracker-release/release-v*/jailer-v*[!.debug] /opt/firecracker/jailer | |
| chmod +x /opt/firecracker/firecracker | |
| chmod +x /opt/firecracker/jailer | |
| find /opt | |
| - name: "Build custom runtime" | |
| run: | | |
| sudo apt update | |
| sudo apt install -y debootstrap ndppd acl cloud-image-utils qemu-utils qemu-system-x86 | |
| cd runtimes/aleph-debian-12-python && sudo ./create_disk_image.sh && cd ../.. | |
| - name: "Build example volume" | |
| run: | | |
| cd examples/volumes && bash build_squashfs.sh | |
| # Unit tests create and delete network interfaces, and therefore require to run as root | |
| - name: Run unit tests | |
| run: | | |
| sudo python3 -m pip install hatch hatch-vcs coverage | |
| sudo hatch run testing:test-cov | |
| sudo hatch run testing:cov | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| slug: aleph-im/aleph-vm | |
| code-quality-shell: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Workaround github issue https://github.com/actions/runner-images/issues/7192 | |
| run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc | |
| - name: Install required system packages only for Ubuntu Linux | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y shellcheck | |
| - name: Run Shellcheck on all shell scripts | |
| run: | | |
| find ./ -type f -name "*.sh" -exec shellcheck {} \; | |
| run_on_droplet: | |
| name: "Test Droplet with ${{ matrix.os_config.os_name }}-${{ matrix.check_vm.alias }}" | |
| runs-on: ubuntu-latest | |
| concurrency: "${{ matrix.os_config.concurrency_group }}-${{ matrix.check_vm.alias }}" | |
| timeout-minutes: 10 | |
| needs: | |
| - tests-python | |
| - code-quality-shell | |
| strategy: | |
| matrix: | |
| # Check compatibility with all supported OSes. | |
| os_config: | |
| - os_name: "Debian 11" | |
| os_image: "debian-11-x64" | |
| alias: "debian-11" | |
| package_build_command: "all-podman-debian-11" | |
| package_name: "aleph-vm.debian-11.deb" | |
| concurrency_group: "droplet-aleph-vm-debian-11" | |
| - os_name: "Debian 12" | |
| os_image: "debian-12-x64" | |
| alias: "debian-12" | |
| package_build_command: "all-podman-debian-12" | |
| package_name: "aleph-vm.debian-12.deb" | |
| concurrency_group: "droplet-aleph-vm-debian-12" | |
| - os_name: "Ubuntu 22.04" | |
| os_image: "ubuntu-22-04-x64" | |
| alias: "ubuntu-22-04" | |
| package_build_command: "all-podman-ubuntu-2204" | |
| package_name: "aleph-vm.ubuntu-22.04.deb" | |
| concurrency_group: "droplet-aleph-vm-ubuntu-22-04" | |
| # Check compatibility with all supported runtimes. | |
| check_vm: | |
| - alias: "runtime-6770" # Old runtime, using Debian 11 | |
| item_hash: "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8" | |
| query_params: "?retro-compatibility=true" | |
| - alias: "runtime-3fc0" # New runtime, using Debian 12 | |
| item_hash: "3fc0aa9569da840c43e7bd2033c3c580abb46b007527d6d20f2d4e98e867f7af" | |
| query_params: "" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install doctl | |
| uses: digitalocean/action-doctl@v2 | |
| with: | |
| token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} | |
| - name: Setup SSH private key | |
| run: | | |
| mkdir ~/.ssh | |
| echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519 | |
| chmod 0700 ~/.ssh | |
| chmod 0600 ~/.ssh/id_ed25519 | |
| env: | |
| DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }} | |
| - name: Create the Droplet | |
| run: | | |
| doctl compute droplet create \ | |
| --image ${{ matrix.os_config.os_image }} \ | |
| --size c-4 \ | |
| --region ams3 \ | |
| --vpc-uuid 5976b7bd-4417-49e8-8522-672aaa920c30 \ | |
| --enable-ipv6 \ | |
| --ssh-keys ab:2b:25:16:46:6f:25:d0:80:63:e5:be:67:04:cb:64 \ | |
| aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} | |
| - name: Build Package | |
| run: | | |
| echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> packaging/aleph-vm/etc/aleph-vm/supervisor.env | |
| echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> packaging/aleph-vm/etc/aleph-vm/supervisor.env | |
| echo ALEPH_VM_CHECK_FASTAPI_VM_ID=${{ matrix.check_vm.item_hash }} >> packaging/aleph-vm/etc/aleph-vm/supervisor.env | |
| echo ALEPH_VM_SENTRY_DSN=${{ secrets.SENTRY_DSN }} >> packaging/aleph-vm/etc/aleph-vm/supervisor.env | |
| cd packaging && make ${{ matrix.os_config.package_build_command }} && cd .. | |
| ls packaging/target | |
| - name: Wait for the system to setup and boot | |
| run: | | |
| export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)" | |
| until ssh-keyscan -H ${DROPLET_IPV4}; do sleep 1; done | |
| timeout-minutes: 3 | |
| - name: Install Aleph-VM on the Droplet | |
| run: | | |
| export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)" | |
| ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts | |
| # Wait a few seconds for DigitalOcean to setup the Droplet using apt, which conflicts with our comands: | |
| sleep 5 | |
| # Wait for /var/lib/apt/lists/lock to be unlocked on the remote host via SSH. | |
| while ssh root@${DROPLET_IPV4} lsof /var/lib/apt/lists/lock; do sleep 1; done | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 update" | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 upgrade -y" | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y docker.io apparmor-profiles" | |
| ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha" | |
| scp packaging/target/${{ matrix.os_config.package_name }} root@${DROPLET_IPV4}:/opt | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=60 install -y /opt/${{ matrix.os_config.package_name }}" | |
| # Allow some time for IPFS Kubo to start | |
| sleep 5 | |
| - name: Test Aleph-VM on the Droplet | |
| id: test-aleph-vm | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)" | |
| curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/about/usage/system" | |
| curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi${{ matrix.check_vm.query_params }}" | |
| - name: Test Aleph-VM on the Droplet again restarting the server first | |
| if: steps.test-aleph-vm.outcome == 'failure' | |
| run: | | |
| export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)" | |
| # If the first execution fails, restart supervisor and try again | |
| ssh root@${DROPLET_IPV4} "systemctl restart aleph-vm-supervisor" | |
| sleep 5 | |
| curl --retry 5 --max-time 10 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi${{ matrix.check_vm.query_params }}" | |
| - name: Schedule an instance on the Droplet by faking a call from the scheduler | |
| run: | | |
| export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)" | |
| curl --retry 5 --max-time 10 --fail -X POST -H "Content-Type: application/json" \ | |
| -H "X-Auth-Signature: test" \ | |
| -d '{"persistent_vms": [], "instances": ["${{ matrix.check_vm.item_hash }}"]}' \ | |
| "http://${DROPLET_IPV4}:4020/control/allocations" | |
| - name: Export aleph logs | |
| if: always() | |
| run: | | |
| export DROPLET_IPV4="$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)" | |
| ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor" | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| doctl compute droplet delete -f aleph-vm-ci-${{ matrix.os_config.alias }}-${{ matrix.check_vm.alias }} |