From e3acf8db9f0b140fcb4ada24792a9e2c374fb093 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Mon, 28 Oct 2024 11:29:06 +1100 Subject: [PATCH] tests: add test for pivot_root in initramfs support The only way to run in the proper initramfs is to start a VM using a custom initrd that runs runc. This should be a fairly reasonable smoke-test that matches what minikube and kata do. Unfortunately, running the right qemu for the native architecture on various distros is a little different, so we need a helper function to get it to work on both Debian and AlmaLinux. Signed-off-by: Aleksa Sarai --- .cirrus.yml | 2 +- .github/workflows/test.yml | 2 +- Dockerfile | 2 + Makefile | 1 + tests/integration/helpers.bash | 16 +++- tests/integration/initramfs.bats | 151 +++++++++++++++++++++++++++++++ 6 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 tests/integration/initramfs.bats diff --git a/.cirrus.yml b/.cirrus.yml index e63ef51cf1a..97e9bdddd09 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -79,7 +79,7 @@ task: CIRRUS_WORKING_DIR: /home/runc GO_VERSION: "1.23" BATS_VERSION: "v1.9.0" - RPMS: gcc git iptables jq glibc-static libseccomp-devel make criu fuse-sshfs container-selinux + RPMS: gcc git iptables jq qemu-kvm glibc-static libseccomp-devel make criu fuse-sshfs container-selinux # yamllint disable rule:key-duplicates matrix: DISTRO: almalinux-8 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9466d5996be..95369561267 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -120,7 +120,7 @@ jobs: - name: install deps run: | sudo apt update - sudo apt -y install libseccomp-dev sshfs uidmap + sudo apt -y install cpio libseccomp-dev qemu-kvm sshfs uidmap - name: install CRIU if: ${{ matrix.criu == '' }} diff --git a/Dockerfile b/Dockerfile index f51f5956c85..7df28fcb5aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ RUN KEYFILE=/usr/share/keyrings/criu-repo-keyring.gpg; \ criu \ gcc \ gcc-multilib \ + cpio \ curl \ gawk \ gperf \ @@ -24,6 +25,7 @@ RUN KEYFILE=/usr/share/keyrings/criu-repo-keyring.gpg; \ kmod \ pkg-config \ python3-minimal \ + qemu-kvm \ sshfs \ sudo \ uidmap \ diff --git a/Makefile b/Makefile index 6bdd031addc..0e5cbd39392 100644 --- a/Makefile +++ b/Makefile @@ -172,6 +172,7 @@ localunittest: all integration: runcimage $(CONTAINER_ENGINE) run $(CONTAINER_ENGINE_RUN_FLAGS) \ -t --privileged --rm \ + -v /boot:/boot:ro \ -v /lib/modules:/lib/modules:ro \ -v $(CURDIR):/go/src/$(PROJECT) \ $(RUNC_IMAGE) make localintegration TESTPATH="$(TESTPATH)" diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash index 85f1fb0a558..0c70616313b 100755 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -13,6 +13,7 @@ eval "$IMAGES" unset IMAGES : "${RUNC:="${INTEGRATION_ROOT}/../../runc"}" +: "${RUNC_STATIC:="${INTEGRATION_ROOT}/../../runc.static"}" RECVTTY="${INTEGRATION_ROOT}/../../tests/cmd/recvtty/recvtty" SD_HELPER="${INTEGRATION_ROOT}/../../tests/cmd/sd-helper/sd-helper" SECCOMP_AGENT="${INTEGRATION_ROOT}/../../tests/cmd/seccompagent/seccompagent" @@ -39,18 +40,25 @@ ARCH=$(uname -m) # Seccomp agent socket. SECCCOMP_AGENT_SOCKET="$BATS_TMPDIR/seccomp-agent.sock" -# Wrapper for runc. -function runc() { - run __runc "$@" +function sane_run() { + local cmd="$1" + shift + + run "$cmd" "$@" # Some debug information to make life easier. bats will only print it if the # test failed, in which case the output is useful. # shellcheck disable=SC2154 - echo "$(basename "$RUNC") $* (status=$status):" >&2 + echo "$cmd $* (status=$status):" >&2 # shellcheck disable=SC2154 echo "$output" >&2 } +# Wrapper for runc. +function runc() { + sane_run __runc "$@" +} + # Raw wrapper for runc. function __runc() { "$RUNC" ${RUNC_USE_SYSTEMD+--systemd-cgroup} \ diff --git a/tests/integration/initramfs.bats b/tests/integration/initramfs.bats new file mode 100644 index 00000000000..28639a82c53 --- /dev/null +++ b/tests/integration/initramfs.bats @@ -0,0 +1,151 @@ +#!/usr/bin/env bats + +load helpers + +function setup() { + INITRAMFS_ROOT="$(mktemp -d "$BATS_RUN_TMPDIR/runc-initramfs.XXXXXX")" +} + +function teardown() { + [ -v INITRAMFS_ROOT ] && rm -rf "$INITRAMFS_ROOT" +} + +function find_vmlinuz() { + shopt -s nullglob + candidates=( + /boot/vmlinuz + /boot/vmlinuz-$(uname -r)* + /usr/lib*/modules/$(uname -r)/vmlinu* # vmlinuz / vmlinux.{gz,xz,*} + ) + shopt -u nullglob + + for candidate in "${candidates[@]}"; do + [ -e "$candidate" ] || continue + echo "$candidate" + return 0 + done + + skip "could not find host vmlinuz kernel" +} + +function qemu_native() { + # Different distributions put qemu-kvm in different locations and with + # different names. Debian and Ubuntu have a "kvm" binary, while AlmaLinux + # has /usr/libexec/qemu-kvm. + qemu_candidates=("kvm" "qemu-kvm" "/usr/libexec/qemu-kvm") + + qemu_binary= + for candidate in "${qemu_candidates[@]}"; do + "$candidate" -help &>/dev/null || continue + qemu_binary="$candidate" + break + done + # TODO: Maybe we should also try to call qemu-system-FOO for the current + # architecture if qemu-kvm is missing? + [ -n "$qemu_binary" ] || skip "could not find qemu-kvm binary" + + machine_args=() + case "$(go env GOARCH)" in + 386 | amd64) + machine_args=("-machine" "pc") + ;; + arm | arm64) + machine_args=("-machine" "virt") + ;; + *) + echo "could not figure out -machine argument for qemu -- using default" >&2 + ;; + esac + + sane_run "$qemu_binary" "${machine_args[@]}" "$@" + if [ "$status" -ne 0 ]; then + # To help with debugging, output the set of valid machine values. + "$qemu_binary" -machine help >&2 + fi +} + +@test "runc run [initramfs + pivot_root]" { + requires root + + KERNEL="$(find_vmlinuz)" + + # Configure our minimal initrd. + mkdir -p "$INITRAMFS_ROOT/initrd" + pushd "$INITRAMFS_ROOT/initrd" + + # Use busybox as a base for our initrd. + tar --exclude './dev/*' -xf "$BUSYBOX_IMAGE" + # Make sure that "sh" and "poweroff" are installed, otherwise qemu will + # boot loop when init stops. + [ -x ./bin/sh ] || skip "busybox image is missing /bin/sh" + [ -x ./bin/poweroff ] || skip "busybox image is missing /bin/poweroff" + + # Copy the runc binary into the container. In theory we would prefer to + # copy a static binary, but some distros (like openSUSE) don't ship + # libseccomp-static so requiring a static build for any integration test + # run wouldn't work. Instead, we copy all of the library dependencies into + # the rootfs (note that we also have to copy ld-linux-*.so because runc was + # probably built with a newer glibc than the one in our busybox image. + cp "$RUNC" ./bin/runc + readarray -t runclibs \ + <<<"$(ldd "$RUNC" | grep -Eo '/[^ ]*lib[^ ]*.so.[^ ]*')" + mkdir -p ./lib{,64} + cp -vt ./lib/ "${runclibs[@]}" + cp -vt ./lib64/ "${runclibs[@]}" + + # Create a container bundle using the same busybox image. + mkdir -p ./run/bundle + pushd ./run/bundle + mkdir -p rootfs + tar --exclude './dev/*' -C rootfs -xf "$BUSYBOX_IMAGE" + runc spec + update_config '.process.args = ["/bin/echo", "hello from inside the container"]' + popd + + # Build a custom /init script. + cat >./init <<-EOF + #!/bin/sh + + set -x + echo "==START INIT SCRIPT==" + + mkdir -p /proc /sys + mount -t proc proc /proc + mkdir -p /sys + mount -t sysfs sysfs /sys + + mkdir -p /sys/fs/cgroup + mount -t cgroup2 cgroup2 /sys/fs/cgroup + + mkdir -p /tmp + mount -t tmpfs tmpfs /tmp + + mkdir -p /dev + mount -t devtmpfs devtmpfs /dev + mkdir -p /dev/pts + mount -t devpts -o newinstance devpts /dev/pts + mkdir -p /dev/shm + mount --bind /tmp /dev/shm + + runc run -b /run/bundle ctr + + echo "==END INIT SCRIPT==" + poweroff -f + EOF + chmod +x ./init + + find . | cpio -o -H newc >"$INITRAMFS_ROOT/initrd.cpio" + popd + + # Now we can just run the image (use qemu-kvm so that we run on the same + # architecture as the host system). We can just reuse the host kernel. + qemu_native \ + -initrd "$INITRAMFS_ROOT/initrd.cpio" \ + -kernel "$KERNEL" \ + -m 512M \ + -nographic -append console=ttyS0 + [ "$status" -eq 0 ] + [[ "$output" = *"==START INIT SCRIPT=="* ]] + [[ "$output" = *"hello from inside the container"* ]] + [[ "$output" = *"==END INIT SCRIPT=="* ]] +}