From a180c591684570d6c7e56de87dcd61605d7cf9ce Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Wed, 6 Sep 2023 11:14:23 +0800 Subject: [PATCH] ci: Add test cases for CoCo image pulling without forked containerd Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. Fixes #5763 Depends: https://github.com/kata-containers/kata-containers/pull/7688 https://github.com/kata-containers/kata-containers/pull/7676 Signed-off-by: ChengyuZhu6 --- Makefile | 2 +- integration/confidential/lib.sh | 80 +++++++++ .../confidential/fixtures/pod-config.yaml.in | 2 +- .../image_pulling_with_snapshotter.bats | 160 ++++++++++++++++++ 4 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 integration/kubernetes/confidential/image_pulling_with_snapshotter.bats diff --git a/Makefile b/Makefile index 48b892219..8b8ebfc94 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ cc-containerd: # Run the Confidential Containers tests for kubernetes. cc-kubernetes: bash -f .ci/install_bats.sh - K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \ + K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" confidential/image_pulling_with_snapshotter.bats" \ bash integration/kubernetes/run_kubernetes_tests.sh # Run the Confidential Containers AMD SEV specific tests. diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 6e0873d05..3cd78560c 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" +NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-coco-host-sharing.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-coco-guest-pulling.toml" +NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" # Toggle between true and false the service_offload configuration of # the Kata agent. @@ -223,6 +227,8 @@ configure_cc_containerd() { sudo tee -a "$containerd_conf_file" fi + configure_containerd_for_nydus_snapshotter "$containerd_conf_file" + sudo systemctl restart containerd if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then die "containerd seems not operational after reconfigured" @@ -445,3 +451,77 @@ EOF EOF fi } + +############################################################################### + +# remote-snapshotter + +configure_remote_snapshotter() { + case "${SNAPSHOTTER:-}" in + "nydus") + configure_nydus_snapshotter + ;; + *) ;; + + esac +} + +check_containerd_version() { + containerd_version=$(containerd --version | awk '{print $3}') + if echo $containerd_version | grep "^v1.7"; then + return 1 + else + return 0 + fi +} + +configure_containerd_for_nydus_snapshotter() { + local containerd_config="$1" + sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config" + if check_containerd_version; then + sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$containerd_config" + else + sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config" + fi +} + +kill_nydus_snapshotter_process() { + echo "Kill nydus snapshotter" + bin="containerd-nydus-grpc" + sudo kill -9 $(pidof $bin) || true +} + +restore_containerd_config() { + local saved_containerd_conf_file="$1" + echo "Restore containerd to pre-test state." + if [ -f "$saved_containerd_conf_file" ]; then + systemctl stop containerd || true + sleep 5 + sudo mv -f "$saved_containerd_conf_file" "/etc/containerd/config.toml" + systemctl start containerd || true + fi +} + +remove_test_image() { + local test_image = "$1" + sudo crictl rmi "$1" + pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))') + sudo crictl rmi "$pause_name" +} + +restart_nydus_snapshotter() { + kill_nydus_snapshotter_process + echo "Restart nydus snapshotter" + sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" & +} + +configure_nydus_snapshotter() { + echo "Configure nydus snapshotter" + if [ "$EXPORT_MODE" == "image_guest_pull" ]; then + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG" + else + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" + sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG" + fi + restart_nydus_snapshotter +} \ No newline at end of file diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in index 3c8e9d3c1..1d0b7def0 100644 --- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in +++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in @@ -5,7 +5,7 @@ apiVersion: v1 kind: Pod metadata: - name: busybox-cc + name: busybox-cc$INDEX spec: runtimeClassName: $RUNTIMECLASS containers: diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats new file mode 100644 index 000000000..c199c5f2a --- /dev/null +++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats @@ -0,0 +1,160 @@ +#!/usr/bin/env bats +# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" + +tag_suffix="" +if [ "$(uname -m)" != "x86_64" ]; then + tag_suffix="-$(uname -m)" +fi + +# Images used on the tests. + +image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}" + +original_kernel_params=$(get_kernel_params) +# Allow to configure the runtimeClassName on pod configuration. +RUNTIMECLASS="${RUNTIMECLASS:-kata}" +test_tag="[cc][agent][kubernetes][containerd]" + +# Create the test pod. +# +# Note: the global $sandbox_name, $pod_config should be set +# already. It also relies on $CI and $DEBUG exported by CI scripts or +# the developer, to decide how to set debug flags. +# +create_test_pod() { + local pod_config="$1" + # On CI mode we only want to enable the agent debug for the case of + # the test failure to obtain logs. + if [ "${CI:-}" == "true" ]; then + enable_full_debug + elif [ "${DEBUG:-}" == "true" ]; then + enable_full_debug + enable_agent_console + fi + + echo "Create the test sandbox" + echo "Pod config is: $pod_config" + kubernetes_create_cc_pod $pod_config +} + +# Create a pod configuration out of a template file. +# +# Parameters: +# $1 - the container image. +# Return: +# the path to the configuration file. The caller should not care about +# its removal afterwards as it is created under the bats temporary +# directory. +# +# Environment variables: +# RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS. +# +new_pod_config() { + local base_config="${FIXTURES_DIR}/pod-config.yaml.in" + local image="$1" + local index="$2" + + local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX") + IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config" + echo "$new_config" +} + +setup() { + start_date=$(date +"%Y-%m-%d %H:%M:%S") + + pod_config="$(new_pod_config "$image_simple_signed")" + pod_id="" + + kubernetes_delete_all_cc_pods_if_any_exists || true + kill_nydus_snapshotter_process + + echo "Prepare containerd for Confidential Container" + SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" + configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" + + echo "Reconfigure Kata Containers" + switch_image_service_offload off + clear_kernel_params + add_kernel_params "${original_kernel_params}" + + setup_proxy + switch_measured_rootfs_verity_scheme none + configure_containerd_for_nydus_snapshotter +} + +# Check the logged messages on host have a given message. +# Parameters: +# $1 - the message +# +# Note: get the logs since the global $start_date. +# +assert_logs_contain() { + local message="$1" + # Note: with image-rs we get more that the default 1000 lines of logs + journalctl -x -t kata --since "$start_date" -n 100000 | grep "$message" +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod "$pod_config" + fi +} + +@test "$test_tag Test can create two pods with pulling the image only once" { + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + + pod_config_1="$(new_pod_config "$image_unsigned_protected" "1")" + echo $pod_config_1 + create_test_pod $pod_config_1 + pod_config_2="$(new_pod_config "$image_unsigned_protected" "2")" + echo $pod_config_2 + create_test_pod $pod_config_2 + + pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l) + [ ${pull_times} -eq 1 ] + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + switch_image_service_offload on + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod $pod_config + fi +} + +teardown() { + # Print the logs and cleanup resources. + echo "-- Kata logs:" + sudo journalctl -xe -t kata --since "$start_date" -n 100000 + + remove_test_image "$image_unsigned_protected" + kill_nydus_snapshotter_process + + # Allow to not destroy the environment if you are developing/debugging + # tests. + if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then + echo "Leaving changes and created resources untouched" + return + fi + + kubernetes_delete_all_cc_pods_if_any_exists || true + clear_kernel_params + add_kernel_params "${original_kernel_params}" + switch_image_service_offload off + disable_full_debug + restore_containerd_config "$SAVED_CONTAINERD_CONF_FILE" +}