From 1138d9a86abed995ab49f4fe1f8ef4255e481824 Mon Sep 17 00:00:00 2001
From: ChengyuZhu6 <chengyu.zhu@intel.com>
Date: Wed, 6 Sep 2023 11:14:23 +0800
Subject: [PATCH] ci: Add test cases for CoCo image pulling without forked
 containerd

Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo.

Fixes #5763

Depends:
https://github.com/kata-containers/kata-containers/pull/7688
https://github.com/kata-containers/kata-containers/pull/7676

Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>
---
 integration/confidential/lib.sh               | 265 +++++++++++-------
 .../confidential/fixtures/pod-config.yaml.in  |   2 +-
 .../image_pulling_with_snapshotter.bats       | 157 +++++++++++
 3 files changed, 326 insertions(+), 98 deletions(-)
 create mode 100644 integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh
index 9a085c638..2286fea75 100644
--- a/integration/confidential/lib.sh
+++ b/integration/confidential/lib.sh
@@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
 source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
 FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
 SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
+NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc"
+NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-tarfs.toml"
+NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-guest.toml"
+NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
 
 # Toggle between true and false the service_offload configuration of
 # the Kata agent.
@@ -29,18 +33,18 @@ switch_image_service_offload() {
 	load_runtime_config_path
 
 	case "$1" in
-		"on")
-			sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
-				"$RUNTIME_CONFIG_PATH"
-			;;
-		"off")
-			sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
-				"$RUNTIME_CONFIG_PATH"
-
-			;;
-		*)
-			die "Unknown option '$1'"
-			;;
+	"on")
+		sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
+			"$RUNTIME_CONFIG_PATH"
+		;;
+	"off")
+		sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
+			"$RUNTIME_CONFIG_PATH"
+
+		;;
+	*)
+		die "Unknown option '$1'"
+		;;
 	esac
 }
 
@@ -66,13 +70,13 @@ switch_measured_rootfs_verity_scheme() {
 	load_runtime_config_path
 
 	case "$1" in
-		"dm-verity"|"none")
-			sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
-				"$RUNTIME_CONFIG_PATH"
-			;;
-		*)
-			die "Unknown option '$1'"
-			;;
+	"dm-verity" | "none")
+		sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
+			"$RUNTIME_CONFIG_PATH"
+		;;
+	*)
+		die "Unknown option '$1'"
+		;;
 	esac
 }
 
@@ -110,8 +114,8 @@ add_kernel_params() {
 get_kernel_params() {
 	load_runtime_config_path
 
-        local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
-                "$RUNTIME_CONFIG_PATH")
+	local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
+		"$RUNTIME_CONFIG_PATH")
 	echo "$kernel_params"
 }
 
@@ -203,13 +207,13 @@ configure_cc_containerd() {
 	# restart containerd because it might be in an inconsistent state here.
 	sudo systemctl stop containerd
 	sleep 5
-	[ -n "$saved_containerd_conf_file" ] && \
+	[ -n "$saved_containerd_conf_file" ] &&
 		sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
 	sudo systemctl start containerd
 	waitForProcess 30 5 "sudo crictl info >/dev/null"
 
 	# Ensure the cc CRI handler is set.
-	local cri_handler=$(sudo crictl info | \
+	local cri_handler=$(sudo crictl info |
 		jq '.config.containerd.runtimes.kata.cri_handler')
 	if [[ ! "$cri_handler" =~ cc ]]; then
 		sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
@@ -219,8 +223,8 @@ configure_cc_containerd() {
 	if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
 		echo "    [plugins.cri.cni]
 		  # conf_dir is the directory in which the admin places a CNI conf.
-		  conf_dir = \"/etc/cni/net.d\"" | \
-			  sudo tee -a "$containerd_conf_file"
+		  conf_dir = \"/etc/cni/net.d\"" |
+			sudo tee -a "$containerd_conf_file"
 	fi
 
 	sudo systemctl restart containerd
@@ -286,38 +290,36 @@ setup_cosign_signatures_files() {
 
 	# Set-up required files in guest image
 	case "${AA_KBC:-}" in
-		"offline_fs_kbc")
-			add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
-			cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
-			;;
-		"cc_kbc")
-			# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
-			# by the service, as well as the one configured in the Kata Containers rootfs.
-
-			CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
-			CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
-			add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
-			;;
-		*)
-			;;
+	"offline_fs_kbc")
+		add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
+		cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
+		;;
+	"cc_kbc")
+		# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
+		# by the service, as well as the one configured in the Kata Containers rootfs.
+
+		CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
+		CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
+		add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
+		;;
+	*) ;;
 	esac
 }
 
 setup_signature_files() {
 	case "${AA_KBC:-}" in
-		"offline_fs_kbc")
-			setup_offline_fs_kbc_signature_files_in_guest
-			;;
-		"cc_kbc")
-			setup_cc_kbc_signature_files_in_guest
-			;;
-		*)
-			;;
+	"offline_fs_kbc")
+		setup_offline_fs_kbc_signature_files_in_guest
+		;;
+	"cc_kbc")
+		setup_cc_kbc_signature_files_in_guest
+		;;
+	*) ;;
 	esac
 }
 
 # In case the tests run behind a firewall where images needed to be fetched
-# through a proxy. 
+# through a proxy.
 # Note: With measured rootfs enabled, we can not set proxy through
 # agent config file.
 setup_proxy() {
@@ -349,8 +351,8 @@ setup_credentials_files() {
 
 	dest_dir="$(mktemp -t -d offline-fs-kbc-XXXXXXXX)"
 	dest_file=${dest_dir}/aa-offline_fs_kbc-resources.json
-	auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
-	CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}"
+	auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
+	CREDENTIAL="${auth_json}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" >"${dest_file}"
 	cp_to_guest_img "etc" "${dest_file}"
 }
 
@@ -366,48 +368,48 @@ KBS_DB="${KBS_DB:-simple_kbs}"
 
 # Run the simple-kbs
 simple_kbs_run() {
-  # Retrieve simple-kbs repo and tag from versions.yaml
-  local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
-  local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
-
-  # Cleanup and create installation directory
-  esudo rm -rf "${SIMPLE_KBS_DIR}" 
-  mkdir -p "${SIMPLE_KBS_DIR}"
-  pushd "${SIMPLE_KBS_DIR}"
-
-  # Clone and run
-  git clone "${simple_kbs_url}" --branch main
-  pushd simple-kbs
-
-  # Checkout, build and start
-  git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
-  esudo docker-compose build
-  esudo docker-compose up -d
-
-  # Wait for simple-kbs to start
-  waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
-  popd
-  
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
-
-  # Confirm connection to the database is possible
-  waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
-  popd
+	# Retrieve simple-kbs repo and tag from versions.yaml
+	local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
+	local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
+
+	# Cleanup and create installation directory
+	esudo rm -rf "${SIMPLE_KBS_DIR}"
+	mkdir -p "${SIMPLE_KBS_DIR}"
+	pushd "${SIMPLE_KBS_DIR}"
+
+	# Clone and run
+	git clone "${simple_kbs_url}" --branch main
+	pushd simple-kbs
+
+	# Checkout, build and start
+	git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
+	esudo docker-compose build
+	esudo docker-compose up -d
+
+	# Wait for simple-kbs to start
+	waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
+	popd
+
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
+
+	# Confirm connection to the database is possible
+	waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
+	popd
 }
 
 # Stop simple-kbs and database containers
 simple_kbs_stop() {
-  (cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
+	(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
 }
 
 # Delete all test inserted data in the simple-kbs
 simple_kbs_delete_data() {
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
 
-  # Delete all data with 'id = 10'
-  mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	# Delete all data with 'id = 10'
+	mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
     DELETE FROM secrets WHERE id = 10;
     DELETE FROM policy WHERE id = 10;
 EOF
@@ -415,28 +417,97 @@ EOF
 
 # Get the ip of the simple-kbs database docker container
 simple_kbs_get_db_ip() {
-  esudo docker network inspect simple-kbs_default \
-    | jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' \
-    | sed "s|/.*$||g"
+	esudo docker network inspect simple-kbs_default |
+		jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' |
+		sed "s|/.*$||g"
 }
 
 # Add key and keyset to database
 # If measurement is provided, add policy with measurement to database
 simple_kbs_add_key_to_db() {
-  local encryption_key="${1}"
-  local measurement="${2}"
-  
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
-
-  if [ -n "${measurement}" ]; then
-    mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	local encryption_key="${1}"
+	local measurement="${2}"
+
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
+
+	if [ -n "${measurement}" ]; then
+		mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
       INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', 10);
       INSERT INTO policy VALUES (10, '["${measurement}"]', '[]', 0, 0, '[]', now(), NULL, 1);
 EOF
-  else
-    mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	else
+		mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
       INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', NULL);
 EOF
-  fi
+	fi
+}
+
+###############################################################################
+
+# remote-snapshotter
+
+configure_remote_snapshotter() {
+	case "${SNAPSHOTTER:-}" in
+	"nydus")
+		configure_nydus_snapshotter
+		;;
+	*) ;;
+
+	esac
+}
+
+check_containerd_version() {
+	containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
+	if echo $containerd_version | grep -q "^v1.7"; then
+		return 1
+	else
+		return 0
+	fi
+}
+
+configure_containerd_for_nydus_snapshotter() {
+	sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG"
+	if check_containerd_version; then
+		sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\  snapshotter = "nydus"\n' "$CONTAINERD_CONFIG"
+	else
+		sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
+	fi
+}
+
+remove_nydus_snapshotter_from_containerd() {
+	sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG"
+	if check_containerd_version; then
+		sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG"
+	else
+		sudo sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
+	fi
+	bin="containerd-nydus-grpc"
+	kill -9 $(pidof $bin) || true
+}
+
+remove_test_image() {
+	local test_image = "$1"
+	sudo crictl rmi "$1"
+	pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
+	sudo crictl rmi "$pause_name"
+}
+
+restart_nydus_snapshotter() {
+	echo "Kill nydus snapshotter"
+	bin="containerd-nydus-grpc"
+	sudo kill -9 $(pidof $bin) || true
+	echo "Restart nydus snapshotter"
+	sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" &
+}
+
+configure_nydus_snapshotter() {
+	echo "Configure nydus snapshotter"
+	if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
+		NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
+	else
+		NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
+		sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
+	fi
+	restart_nydus_snapshotter
 }
diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
index 3c8e9d3c1..1d0b7def0 100644
--- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
+++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
@@ -5,7 +5,7 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: busybox-cc
+  name: busybox-cc$INDEX
 spec:
   runtimeClassName: $RUNTIMECLASS
   containers:
diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats
new file mode 100644
index 000000000..8776146be
--- /dev/null
+++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats
@@ -0,0 +1,157 @@
+#!/usr/bin/env bats
+# Copyright (c) 2022 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+
+tag_suffix=""
+if [ "$(uname -m)" != "x86_64" ]; then
+    tag_suffix="-$(uname -m)"
+fi
+
+# Images used on the tests.
+
+image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
+
+original_kernel_params=$(get_kernel_params)
+# Allow to configure the runtimeClassName on pod configuration.
+RUNTIMECLASS="${RUNTIMECLASS:-kata}"
+test_tag="[cc][agent][kubernetes][containerd]"
+
+# Create the test pod.
+#
+# Note: the global $sandbox_name, $pod_config should be set
+# 	already. It also relies on $CI and $DEBUG exported by CI scripts or
+# 	the developer, to decide how to set debug flags.
+#
+create_test_pod() {
+    local pod_config="$1"
+    # On CI mode we only want to enable the agent debug for the case of
+    # the test failure to obtain logs.
+    if [ "${CI:-}" == "true" ]; then
+        enable_full_debug
+    elif [ "${DEBUG:-}" == "true" ]; then
+        enable_full_debug
+        enable_agent_console
+    fi
+
+    echo "Create the test sandbox"
+    echo "Pod config is: $pod_config"
+    kubernetes_create_cc_pod $pod_config
+}
+
+# Create a pod configuration out of a template file.
+#
+# Parameters:
+#	$1 - the container image.
+# Return:
+# 	the path to the configuration file. The caller should not care about
+# 	its removal afterwards as it is created under the bats temporary
+# 	directory.
+#
+# Environment variables:
+#	RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS.
+#
+new_pod_config() {
+    local base_config="${FIXTURES_DIR}/pod-config.yaml.in"
+    local image="$1"
+    local index="$2"
+
+    local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
+    IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config"
+    echo "$new_config"
+}
+
+setup() {
+    start_date=$(date +"%Y-%m-%d %H:%M:%S")
+
+    pod_config="$(new_pod_config "$image_simple_signed")"
+    pod_id=""
+
+    kubernetes_delete_all_cc_pods_if_any_exists || true
+
+    echo "Prepare containerd for Confidential Container"
+    SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
+    configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
+
+    echo "Reconfigure Kata Containers"
+    switch_image_service_offload off
+    clear_kernel_params
+    add_kernel_params "${original_kernel_params}"
+
+    setup_proxy
+    switch_measured_rootfs_verity_scheme none
+    configure_containerd_for_nydus_snapshotter
+}
+
+# Check the logged messages on host have a given message.
+# Parameters:
+#      $1 - the message
+#
+# Note: get the logs since the global $start_date.
+#
+assert_logs_contain() {
+    local message="$1"
+    # Note: with image-rs we get more that the default 1000 lines of logs
+    journalctl -x -t kata --since "$start_date" -n 100000 | grep "$message"
+}
+
+@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_protected")"
+        echo $pod_config
+        create_test_pod "$pod_config"
+    fi
+}
+
+@test "$test_tag Test can create two pods with pulling the image only once" {
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+
+        pod_config_1="$(new_pod_config "$image_unsigned_protected" "1")"
+        echo $pod_config_1
+        create_test_pod $pod_config_1
+        pod_config_2="$(new_pod_config "$image_unsigned_protected" "2")"
+        echo $pod_config_2
+        create_test_pod $pod_config_2
+
+        pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l)
+        [ ${pull_times} -eq 1 ]
+    fi
+}
+
+@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
+    switch_image_service_offload on
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_protected")"
+        echo $pod_config
+        create_test_pod $pod_config
+    fi
+}
+
+teardown() {
+    # Print the logs and cleanup resources.
+    echo "-- Kata logs:"
+    sudo journalctl -xe -t kata --since "$start_date" -n 100000
+
+    remove_test_image "$image_unsigned_protected"
+
+    # Allow to not destroy the environment if you are developing/debugging
+    # tests.
+    if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then
+        echo "Leaving changes and created resources untouched"
+        return
+    fi
+
+    kubernetes_delete_all_cc_pods_if_any_exists || true
+    clear_kernel_params
+    add_kernel_params "${original_kernel_params}"
+    switch_image_service_offload off
+    remove_nydus_snapshotter_from_containerd || true
+    disable_full_debug
+}