From f9dd8c929c50db2c9f0ad0fa9d79599a6813064c Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Wed, 6 Sep 2023 11:14:23 +0800 Subject: [PATCH] ci: Add test cases for CoCo image pulling without forked containerd Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. Fixes #5763 Depends: https://github.com/kata-containers/kata-containers/pull/7688 https://github.com/kata-containers/kata-containers/pull/7676 Signed-off-by: ChengyuZhu6 --- integration/confidential/lib.sh | 50 ++++++ .../confidential/fixtures/pod-config.yaml.in | 2 +- .../image_pulling_with_snapshotter.bats | 149 ++++++++++++++++++ 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 integration/kubernetes/confidential/image_pulling_with_snapshotter.bats diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 9a085c638..4f38557ae 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,6 +12,9 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" +NYDUS_SNAPSHOTTER_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config.toml" +NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc" +CONTAINERD_CONFIG="/etc/containerd/config.toml" # Toggle between true and false the service_offload configuration of # the Kata agent. @@ -440,3 +443,50 @@ EOF EOF fi } + +############################################################################### + +# remote-snapshotter + +configure_remote_snapshotter() { + case "${SNAPSHOTTER:-}" in + "nydus") + configure_nydus_snapshotter + ;; + *) ;; + + esac +} +check_containerd_version() { + containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1) + if echo $containerd_version | grep -q "^v1.7"; then + return 1 + else + return 0 + fi +} +configure_containerd_for_nydus_snapshotter() { + sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG" + if check_containerd_version; then + sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$CONTAINERD_CONFIG" + else + sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG" + fi +} +remove_nydus_snapshotter_from_containerd() { + sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG" + if check_containerd_version; then + sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG" + else + sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG" + fi +} +restart_nydus_snapshotter() { + echo "Restart nydus snapshotter" +} + +configure_nydus_snapshotter() { + echo "Configure nydus snapshotter" + sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" $NYDUS_SNAPSHOTTER_CONFIG + restart_nydus_snapshotter +} diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in index 3c8e9d3c1..1d0b7def0 100644 --- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in +++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in @@ -5,7 +5,7 @@ apiVersion: v1 kind: Pod metadata: - name: busybox-cc + name: busybox-cc$INDEX spec: runtimeClassName: $RUNTIMECLASS containers: diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats new file mode 100644 index 000000000..d23891131 --- /dev/null +++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats @@ -0,0 +1,149 @@ +#!/usr/bin/env bats +# Copyright (c) 2022 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" + +tag_suffix="" +if [ "$(uname -m)" != "x86_64" ]; then + tag_suffix="-$(uname -m)" +fi + +# Images used on the tests. + +image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}" + +original_kernel_params=$(get_kernel_params) +# Allow to configure the runtimeClassName on pod configuration. +RUNTIMECLASS="${RUNTIMECLASS:-kata}" +test_tag="[cc][agent][kubernetes][containerd]" + +# Create the test pod. +# +# Note: the global $sandbox_name, $pod_config should be set +# already. It also relies on $CI and $DEBUG exported by CI scripts or +# the developer, to decide how to set debug flags. +# +create_test_pod() { + local pod_config="$1" + # On CI mode we only want to enable the agent debug for the case of + # the test failure to obtain logs. + if [ "${CI:-}" == "true" ]; then + enable_full_debug + elif [ "${DEBUG:-}" == "true" ]; then + enable_full_debug + enable_agent_console + fi + + echo "Create the test sandbox" + echo "Pod config is: $pod_config" + kubernetes_create_cc_pod $pod_config +} + +# Create a pod configuration out of a template file. +# +# Parameters: +# $1 - the container image. +# Return: +# the path to the configuration file. The caller should not care about +# its removal afterwards as it is created under the bats temporary +# directory. +# +# Environment variables: +# RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS. +# +new_pod_config() { + local base_config="${FIXTURES_DIR}/pod-config.yaml.in" + local image="$1" + local index="$2" + + local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX") + IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config" + echo "$new_config" +} + +setup() { + start_date=$(date +"%Y-%m-%d %H:%M:%S") + + pod_config="$(new_pod_config "$image_simple_signed")" + pod_id="" + + kubernetes_delete_all_cc_pods_if_any_exists || true + + echo "Prepare containerd for Confidential Container" + SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" + configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" + + echo "Reconfigure Kata Containers" + switch_image_service_offload off + clear_kernel_params + add_kernel_params "${original_kernel_params}" + + setup_proxy + switch_measured_rootfs_verity_scheme none +} + +# Check the logged messages on host have a given message. +# Parameters: +# $1 - the message +# +# Note: get the logs since the global $start_date. +# +assert_logs_contain() { + local message="$1" + # Note: with image-rs we get more that the default 1000 lines of logs + journalctl -x -t kata --since "$start_date" -n 100000 | grep "$message" +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "${SNAPSHOTTER}" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_unprotected")" + echo $pod_config + create_test_pod "$pod_config" + fi +} + +@test "$test_tag Test can create two pods by pulling the image only once" { + if [ "${SNAPSHOTTER}" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter + + pod_config_1="$(new_pod_config "$image_unsigned_unprotected" "1")" + echo $pod_config_1 + create_test_pod $pod_config_1 + pod_config_2="$(new_pod_config "$image_unsigned_unprotected" "2")" + echo $pod_config_2 + create_test_pod $pod_config_2 + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + if [ "${SNAPSHOTTER}" = "nydus" ]; then + switch_image_service_offload on + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter + create_test_pod + fi +} + +teardown() { + # Print the logs and cleanup resources. + echo "-- Kata logs:" + sudo journalctl -xe -t kata --since "$start_date" -n 100000 + + # Allow to not destroy the environment if you are developing/debugging + # tests. + if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then + echo "Leaving changes and created resources untouched" + return + fi + + kubernetes_delete_all_cc_pods_if_any_exists || true + clear_kernel_params + add_kernel_params "${original_kernel_params}" + switch_image_service_offload off + remove_nydus_snapshotter_from_containerd + disable_full_debug +}