ci: Add test cases for CoCo image pulling without forked containerd

Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. 1)image sharing on the host without dm-verity. 2)image sharing on the host with dm-verity. 3)image pulling in the guest with nydus-snapshotter. Fixes #5763 Depends-on:github.com/kata-containers/kata-containers/pull/7676 Depends-on:github.com/confidential-containers/operator#251 Signed-off-by: ChengyuZhu6 <[email protected]>
kata-containers · Sep 11, 2023 · 66c42d9 · 66c42d9
1 parent 6d7723a
commit 66c42d9
Show file tree

Hide file tree

Showing 6 changed files with 183 additions and 4 deletions.
diff --git a/Makefile b/Makefile
@@ -104,7 +104,7 @@ cc-containerd:
 # Run the Confidential Containers tests for kubernetes.
 cc-kubernetes:
 	bash -f .ci/install_bats.sh
-	K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \
+	K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \
 	bash integration/kubernetes/run_kubernetes_tests.sh
 
 # Run the Confidential Containers AMD SEV specific tests.

diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh
@@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
 source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
 FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
 SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
+NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc"
+NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-coco-host-sharing.toml"
+NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-coco-guest-pulling.toml"
+NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
 
 # Toggle between true and false the service_offload configuration of
 # the Kata agent.
@@ -223,6 +227,8 @@ configure_cc_containerd() {
 			  sudo tee -a "$containerd_conf_file"
 	fi
 
+	configure_containerd_for_nydus_snapshotter "$containerd_conf_file"
+
 	sudo systemctl restart containerd
 	if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
 		die "containerd seems not operational after reconfigured"
@@ -445,3 +451,85 @@ EOF
 EOF
   fi
 }
+
+###############################################################################
+
+# remote-snapshotter
+
+configure_remote_snapshotter() {
+	case "${SNAPSHOTTER:-}" in
+	"nydus")
+		configure_nydus_snapshotter
+		;;
+	*) ;;
+
+	esac
+}
+
+is_containerd_support_per_runtime_snapshotter () {
+   containerd_version=$(containerd --version | awk '{print $3}')
+   required_version="v1.7.0"
+   printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet  -V
+}
+
+configure_containerd_for_nydus_snapshotter() {
+	if [ "${SNAPSHOTTER:-}" = "nydus" ]; then
+		local containerd_config="$1"
+		sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config"
+		if is_containerd_support_per_runtime_snapshotter; then
+			sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\  snapshotter = "nydus"' "$containerd_config"
+		else
+			sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config"
+		fi
+	fi
+}
+
+kill_nydus_snapshotter_process() {
+	echo "Kill nydus snapshotter"
+	bin="containerd-nydus-grpc"
+	sudo kill -9 $(pidof $bin) || true
+	kill -9 $(pidof $bin) || true
+	rm -rf "/var/lib/containerd-nydus" || true
+}
+
+restore_containerd_config() {
+	local saved_containerd_conf_file="$1"
+	echo "Restore containerd to pre-test state."
+	if [ -f "$saved_containerd_conf_file" ]; then
+		systemctl stop containerd || true
+		sleep 5
+		sudo mv -f "$saved_containerd_conf_file" "/etc/containerd/config.toml"
+		systemctl start containerd || true
+	fi
+}
+
+remove_test_image() {
+	local test_image="$1"
+	if ctr -n k8s.io i ls | grep -q "$test_image"; then
+		crictl rmi "$1"
+	fi
+
+	pause_name=$(grep 'sandbox_image' /etc/containerd/config.toml | awk -F'"' '{print $2}')
+	if ctr -n k8s.io i ls | grep -q "$pause_name"; then
+		crictl rmi "$pause_name"
+	fi
+}
+
+restart_nydus_snapshotter() {
+	kill_nydus_snapshotter_process || true
+	echo "Restart nydus snapshotter"
+	sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 &
+}
+
+configure_nydus_snapshotter() {
+	if [ "${SNAPSHOTTER:-}" = "nydus" ]; then
+		echo "Configure nydus snapshotter"
+		if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
+			NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
+		else
+			NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
+			sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
+		fi
+		restart_nydus_snapshotter
+	fi
+}
diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
@@ -5,7 +5,7 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: busybox-cc
+  name: busybox-cc$INDEX
 spec:
   runtimeClassName: $RUNTIMECLASS
   containers:

diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats
@@ -0,0 +1,88 @@
+#!/usr/bin/env bats
+# Copyright (c) 2023 Intel Corporation
+# Copyright (c) 2023 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+
+tag_suffix=""
+if [ "$(uname -m)" != "x86_64" ]; then
+    tag_suffix="-$(uname -m)"
+fi
+
+# Images used on the tests.
+
+image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
+
+original_kernel_params=$(get_kernel_params)
+# Allow to configure the runtimeClassName on pod configuration.
+RUNTIMECLASS="${RUNTIMECLASS:-kata}"
+test_tag="[cc][agent][kubernetes][containerd]"
+
+setup() {
+    remove_test_image "$image_unsigned_protected"
+    kill_nydus_snapshotter_process
+    setup_common
+}
+
+@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
+    if [ "$(uname -m)" = s390x ]; then
+        skip "test for s390x as nydus-image doesn't currently support this platform"
+    fi
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_protected")"
+        echo $pod_config
+        create_test_pod "$pod_config"
+    fi
+}
+
+@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" {
+    if [ "$(uname -m)" = s390x ]; then
+        skip "test for s390x as nydus-image doesn't currently support this platform"
+    fi
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_protected")"
+        echo $pod_config
+        create_test_pod "$pod_config"
+    fi
+}
+
+@test "$test_tag Test can create two pods with pulling the image only once with dm-verity enabled" {
+    if [ "$(uname -m)" = s390x ]; then
+        skip "test for s390x as nydus-image doesn't currently support this platform"
+    fi
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+
+        pod_config_1="$(new_pod_config "$image_unsigned_protected" "1")"
+        echo $pod_config_1
+        create_test_pod $pod_config_1
+        pod_config_2="$(new_pod_config "$image_unsigned_protected" "2")"
+        echo $pod_config_2
+        create_test_pod $pod_config_2
+
+        pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l)
+        [ ${pull_times} -eq 1 ]
+    fi
+}
+
+@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
+    switch_image_service_offload on
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_protected")"
+        echo $pod_config
+        create_test_pod $pod_config
+    fi
+}
+
+teardown() {
+    remove_test_image "$image_unsigned_protected"
+    kill_nydus_snapshotter_process
+    teardown_common
+}
diff --git a/integration/kubernetes/confidential/lib.sh b/integration/kubernetes/confidential/lib.sh
@@ -46,7 +46,9 @@ kubernetes_create_cc_pod() {
 	fi
 
 	kubectl apply -f ${config_file}
-	if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
+	pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name')
+	echo $(kubectl get pod "$pod_name" &> /dev/null)
+	if ! kubectl get pod "$pod_name" &> /dev/null; then
 		echo "Failed to create the pod"
 		return 1
 	fi

diff --git a/integration/kubernetes/confidential/tests_common.sh b/integration/kubernetes/confidential/tests_common.sh
@@ -97,8 +97,9 @@ create_test_pod() {
 new_pod_config() {
 	local base_config="${FIXTURES_DIR}/pod-config.yaml.in"
 	local image="$1"
+	local index="${2:-}"
 
 	local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
-	IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config"
+	IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS"  INDEX="$index" envsubst < "$base_config" > "$new_config"
 	echo "$new_config"
 }