Skip to content
This repository has been archived by the owner on Jun 28, 2024. It is now read-only.

Commit

Permalink
ci: Add test cases for CoCo image pulling without forked containerd
Browse files Browse the repository at this point in the history 
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo.

Fixes #5763

Depends:
kata-containers/kata-containers#7688
kata-containers/kata-containers#7676

Signed-off-by: ChengyuZhu6 <[email protected]>
  • Loading branch information
@ChengyuZhu6
ChengyuZhu6 committed Sep 8, 2023
1 parent 021bf58 commit 1138d9a
Show file tree
Hide file tree
Showing 3 changed files with 326 additions and 98 deletions.
265 changes: 168 additions & 97 deletions integration/confidential/lib.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc"
NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-tarfs.toml"
NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-guest.toml"
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"

# Toggle between true and false the service_offload configuration of
# the Kata agent.
Expand All @@ -29,18 +33,18 @@ switch_image_service_offload() {
load_runtime_config_path

case "$1" in
"on")
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
"$RUNTIME_CONFIG_PATH"
;;
"off")
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
"$RUNTIME_CONFIG_PATH"

;;
*)
die "Unknown option '$1'"
;;
"on")
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
"$RUNTIME_CONFIG_PATH"
;;
"off")
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
"$RUNTIME_CONFIG_PATH"

;;
*)
die "Unknown option '$1'"
;;
esac
}

Expand All @@ -66,13 +70,13 @@ switch_measured_rootfs_verity_scheme() {
load_runtime_config_path

case "$1" in
"dm-verity"|"none")
sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
"$RUNTIME_CONFIG_PATH"
;;
*)
die "Unknown option '$1'"
;;
"dm-verity" | "none")
sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
"$RUNTIME_CONFIG_PATH"
;;
*)
die "Unknown option '$1'"
;;
esac
}

Expand Down Expand Up @@ -110,8 +114,8 @@ add_kernel_params() {
get_kernel_params() {
load_runtime_config_path

local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
"$RUNTIME_CONFIG_PATH")
local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
"$RUNTIME_CONFIG_PATH")
echo "$kernel_params"
}

Expand Down Expand Up @@ -203,13 +207,13 @@ configure_cc_containerd() {
# restart containerd because it might be in an inconsistent state here.
sudo systemctl stop containerd
sleep 5
[ -n "$saved_containerd_conf_file" ] && \
[ -n "$saved_containerd_conf_file" ] &&
sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
sudo systemctl start containerd
waitForProcess 30 5 "sudo crictl info >/dev/null"

# Ensure the cc CRI handler is set.
local cri_handler=$(sudo crictl info | \
local cri_handler=$(sudo crictl info |
jq '.config.containerd.runtimes.kata.cri_handler')
if [[ ! "$cri_handler" =~ cc ]]; then
sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
Expand All @@ -219,8 +223,8 @@ configure_cc_containerd() {
if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
echo " [plugins.cri.cni]
# conf_dir is the directory in which the admin places a CNI conf.
conf_dir = \"/etc/cni/net.d\"" | \
sudo tee -a "$containerd_conf_file"
conf_dir = \"/etc/cni/net.d\"" |
sudo tee -a "$containerd_conf_file"
fi

sudo systemctl restart containerd
Expand Down Expand Up @@ -286,38 +290,36 @@ setup_cosign_signatures_files() {

# Set-up required files in guest image
case "${AA_KBC:-}" in
"offline_fs_kbc")
add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
;;
"cc_kbc")
# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
# by the service, as well as the one configured in the Kata Containers rootfs.

CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
;;
*)
;;
"offline_fs_kbc")
add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
;;
"cc_kbc")
# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
# by the service, as well as the one configured in the Kata Containers rootfs.

CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
;;
*) ;;
esac
}

setup_signature_files() {
case "${AA_KBC:-}" in
"offline_fs_kbc")
setup_offline_fs_kbc_signature_files_in_guest
;;
"cc_kbc")
setup_cc_kbc_signature_files_in_guest
;;
*)
;;
"offline_fs_kbc")
setup_offline_fs_kbc_signature_files_in_guest
;;
"cc_kbc")
setup_cc_kbc_signature_files_in_guest
;;
*) ;;
esac
}

# In case the tests run behind a firewall where images needed to be fetched
# through a proxy.
# through a proxy.
# Note: With measured rootfs enabled, we can not set proxy through
# agent config file.
setup_proxy() {
Expand Down Expand Up @@ -349,8 +351,8 @@ setup_credentials_files() {

dest_dir="$(mktemp -t -d offline-fs-kbc-XXXXXXXX)"
dest_file=${dest_dir}/aa-offline_fs_kbc-resources.json
auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}"
auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
CREDENTIAL="${auth_json}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" >"${dest_file}"
cp_to_guest_img "etc" "${dest_file}"
}

Expand All @@ -366,77 +368,146 @@ KBS_DB="${KBS_DB:-simple_kbs}"

# Run the simple-kbs
simple_kbs_run() {
# Retrieve simple-kbs repo and tag from versions.yaml
local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")

# Cleanup and create installation directory
esudo rm -rf "${SIMPLE_KBS_DIR}"
mkdir -p "${SIMPLE_KBS_DIR}"
pushd "${SIMPLE_KBS_DIR}"

# Clone and run
git clone "${simple_kbs_url}" --branch main
pushd simple-kbs

# Checkout, build and start
git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
esudo docker-compose build
esudo docker-compose up -d

# Wait for simple-kbs to start
waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
popd
# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

# Confirm connection to the database is possible
waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
popd
# Retrieve simple-kbs repo and tag from versions.yaml
local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")

# Cleanup and create installation directory
esudo rm -rf "${SIMPLE_KBS_DIR}"
mkdir -p "${SIMPLE_KBS_DIR}"
pushd "${SIMPLE_KBS_DIR}"

# Clone and run
git clone "${simple_kbs_url}" --branch main
pushd simple-kbs

# Checkout, build and start
git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
esudo docker-compose build
esudo docker-compose up -d

# Wait for simple-kbs to start
waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
popd

# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

# Confirm connection to the database is possible
waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
popd
}

# Stop simple-kbs and database containers
simple_kbs_stop() {
(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
}

# Delete all test inserted data in the simple-kbs
simple_kbs_delete_data() {
# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)
# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

# Delete all data with 'id = 10'
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
# Delete all data with 'id = 10'
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
DELETE FROM secrets WHERE id = 10;
DELETE FROM policy WHERE id = 10;
EOF
}

# Get the ip of the simple-kbs database docker container
simple_kbs_get_db_ip() {
esudo docker network inspect simple-kbs_default \
| jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' \
| sed "s|/.*$||g"
esudo docker network inspect simple-kbs_default |
jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' |
sed "s|/.*$||g"
}

# Add key and keyset to database
# If measurement is provided, add policy with measurement to database
simple_kbs_add_key_to_db() {
local encryption_key="${1}"
local measurement="${2}"
# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

if [ -n "${measurement}" ]; then
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
local encryption_key="${1}"
local measurement="${2}"

# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

if [ -n "${measurement}" ]; then
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', 10);
INSERT INTO policy VALUES (10, '["${measurement}"]', '[]', 0, 0, '[]', now(), NULL, 1);
EOF
else
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
else
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', NULL);
EOF
fi
fi
}

###############################################################################

# remote-snapshotter

configure_remote_snapshotter() {
case "${SNAPSHOTTER:-}" in
"nydus")
configure_nydus_snapshotter
;;
*) ;;

esac
}

check_containerd_version() {
containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
if echo $containerd_version | grep -q "^v1.7"; then
return 1
else
return 0
fi
}

configure_containerd_for_nydus_snapshotter() {
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG"
if check_containerd_version; then
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$CONTAINERD_CONFIG"
else
sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
fi
}

remove_nydus_snapshotter_from_containerd() {
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG"
if check_containerd_version; then
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG"
else
sudo sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
fi
bin="containerd-nydus-grpc"
kill -9 $(pidof $bin) || true
}

remove_test_image() {
local test_image = "$1"
sudo crictl rmi "$1"
pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
sudo crictl rmi "$pause_name"
}

restart_nydus_snapshotter() {
echo "Kill nydus snapshotter"
bin="containerd-nydus-grpc"
sudo kill -9 $(pidof $bin) || true
echo "Restart nydus snapshotter"
sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" &
}

configure_nydus_snapshotter() {
echo "Configure nydus snapshotter"
if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
else
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
fi
restart_nydus_snapshotter
}
2 changes: 1 addition & 1 deletion integration/kubernetes/confidential/fixtures/pod-config.yaml.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
apiVersion: v1
kind: Pod
metadata:
name: busybox-cc
name: busybox-cc$INDEX
spec:
runtimeClassName: $RUNTIMECLASS
containers:
Expand Down
Loading

0 comments on commit 1138d9a

Please sign in to comment.