Skip to content

Commit 2ea002a

Browse files
author
ChengyuZhu6
committed
ci: Add test cases for CoCo image pulling without forked containerd
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. Fixes kata-containers#5763 Depends: kata-containers/kata-containers#7688 kata-containers/kata-containers#7676 Signed-off-by: ChengyuZhu6 <[email protected]>
1 parent 021bf58 commit 2ea002a

File tree

3 files changed

+193
-0
lines changed

3 files changed

+193
-0
lines changed

Diff for: ‎integration/confidential/lib.sh

+49
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
1212
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
1313
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
1414
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
15+
NYDUS_SNAPSHOTTER_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config.toml"
16+
CONTAINERD_CONFIG="/etc/containerd/config.toml"
1517

1618
# Toggle between true and false the service_offload configuration of
1719
# the Kata agent.
@@ -440,3 +442,50 @@ EOF
440442
EOF
441443
fi
442444
}
445+
446+
###############################################################################
447+
448+
# remote-snapshotter
449+
450+
configure_remote_snapshotter() {
451+
case "${SNAPSHOTTER:-}" in
452+
"nydus")
453+
configure_nydus_snapshotter
454+
;;
455+
*) ;;
456+
457+
esac
458+
}
459+
check_containerd_version() {
460+
containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
461+
if echo $containerd_version | grep -q "^v1.7"; then
462+
return 1
463+
else
464+
return 0
465+
fi
466+
}
467+
configure_containerd_for_nydus_snapshotter() {
468+
sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG"
469+
if check_containerd_version; then
470+
sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$CONTAINERD_CONFIG"
471+
else
472+
sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
473+
fi
474+
}
475+
remove_nydus_snapshotter_from_containerd() {
476+
sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG"
477+
if check_containerd_version; then
478+
sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG"
479+
else
480+
sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
481+
fi
482+
}
483+
restart_nydus_snapshotter() {
484+
echo "Restart nydus snapshotter"
485+
}
486+
487+
configure_nydus_snapshotter() {
488+
echo "Configure nydus snapshotter"
489+
sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" $NYDUS_SNAPSHOTTER_CONFIG
490+
restart_nydus_snapshotter
491+
}

Diff for: ‎integration/kubernetes/confidential/agent_image.bats

+9
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,15 @@ assert_logs_contain() {
223223
assert_logs_contain 'failed to pull manifest Not authorized'
224224
}
225225

226+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
227+
if [ "${SNAPSHOTTER}" = "nydus" ]; then
228+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter
229+
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
230+
echo $pod_config
231+
create_test_pod
232+
fi
233+
}
234+
226235
teardown() {
227236
# Print the logs and cleanup resources.
228237
echo "-- Kata logs:"

Diff for: ‎integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/usr/bin/env bats
2+
# Copyright (c) 2022 IBM Corporation
3+
#
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
7+
load "${BATS_TEST_DIRNAME}/lib.sh"
8+
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
9+
10+
tag_suffix=""
11+
if [ "$(uname -m)" != "x86_64" ]; then
12+
tag_suffix="-$(uname -m)"
13+
fi
14+
15+
# Images used on the tests.
16+
17+
image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
18+
19+
original_kernel_params=$(get_kernel_params)
20+
# Allow to configure the runtimeClassName on pod configuration.
21+
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
22+
test_tag="[cc][agent][kubernetes][containerd]"
23+
24+
# Create the test pod.
25+
#
26+
# Note: the global $sandbox_name, $pod_config should be set
27+
# already. It also relies on $CI and $DEBUG exported by CI scripts or
28+
# the developer, to decide how to set debug flags.
29+
#
30+
create_test_pod() {
31+
# On CI mode we only want to enable the agent debug for the case of
32+
# the test failure to obtain logs.
33+
if [ "${CI:-}" == "true" ]; then
34+
enable_full_debug
35+
elif [ "${DEBUG:-}" == "true" ]; then
36+
enable_full_debug
37+
enable_agent_console
38+
fi
39+
40+
echo "Create the test sandbox"
41+
echo "Pod config is: $pod_config"
42+
kubernetes_create_cc_pod $pod_config
43+
}
44+
45+
# Create a pod configuration out of a template file.
46+
#
47+
# Parameters:
48+
# $1 - the container image.
49+
# Return:
50+
# the path to the configuration file. The caller should not care about
51+
# its removal afterwards as it is created under the bats temporary
52+
# directory.
53+
#
54+
# Environment variables:
55+
# RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS.
56+
#
57+
new_pod_config() {
58+
local base_config="${FIXTURES_DIR}/pod-config.yaml.in"
59+
local image="$1"
60+
61+
local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
62+
IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config"
63+
echo "$new_config"
64+
}
65+
66+
setup() {
67+
start_date=$(date +"%Y-%m-%d %H:%M:%S")
68+
69+
pod_config="$(new_pod_config "$image_simple_signed")"
70+
pod_id=""
71+
72+
kubernetes_delete_all_cc_pods_if_any_exists || true
73+
74+
echo "Prepare containerd for Confidential Container"
75+
SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
76+
configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
77+
78+
echo "Reconfigure Kata Containers"
79+
switch_image_service_offload off
80+
clear_kernel_params
81+
add_kernel_params "${original_kernel_params}"
82+
83+
setup_proxy
84+
switch_measured_rootfs_verity_scheme none
85+
}
86+
87+
# Check the logged messages on host have a given message.
88+
# Parameters:
89+
# $1 - the message
90+
#
91+
# Note: get the logs since the global $start_date.
92+
#
93+
assert_logs_contain() {
94+
local message="$1"
95+
# Note: with image-rs we get more that the default 1000 lines of logs
96+
journalctl -x -t kata --since "$start_date" -n 100000 | grep "$message"
97+
}
98+
99+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
100+
if [ "${SNAPSHOTTER}" = "nydus" ]; then
101+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter
102+
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
103+
echo $pod_config
104+
create_test_pod
105+
fi
106+
}
107+
108+
@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
109+
if [ "${SNAPSHOTTER}" = "nydus" ]; then
110+
switch_image_service_offload on
111+
EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter
112+
create_test_pod
113+
fi
114+
}
115+
116+
117+
teardown() {
118+
# Print the logs and cleanup resources.
119+
echo "-- Kata logs:"
120+
sudo journalctl -xe -t kata --since "$start_date" -n 100000
121+
122+
# Allow to not destroy the environment if you are developing/debugging
123+
# tests.
124+
if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then
125+
echo "Leaving changes and created resources untouched"
126+
return
127+
fi
128+
129+
kubernetes_delete_all_cc_pods_if_any_exists || true
130+
clear_kernel_params
131+
add_kernel_params "${original_kernel_params}"
132+
switch_image_service_offload off
133+
remove_nydus_snapshotter_from_containerd
134+
disable_full_debug
135+
}

0 commit comments

Comments
 (0)