Skip to content

Commit 8a062f9

Browse files
author
ChengyuZhu6
committed
ci: Add test cases for CoCo image pulling without forked containerd
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. Fixes kata-containers#5763 Depends: kata-containers/kata-containers#7688 kata-containers/kata-containers#7676 Signed-off-by: ChengyuZhu6 <[email protected]>
1 parent 021bf58 commit 8a062f9

File tree

4 files changed

+171
-2
lines changed

4 files changed

+171
-2
lines changed

Diff for: ‎integration/confidential/lib.sh

+68-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
1212
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
1313
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
1414
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
15-
15+
CONTAINERD_CONFIG="/etc/containerd/config.toml"
16+
NYDUS_SNAPSHOTTER_BINARY="/home/zcy/workspace/image_sharing/bin/containerd-nydus-grpc"
17+
NYDUS_SNAPSHOTTER_CONFIG="/etc/nydus/config-tarfs.toml"
18+
NYDUS_SNAPSHOTTER_TARFS_CONFIG="/etc/nydus/config-tarfs.toml"
19+
NYDUS_SNAPSHOTTER_GUEST_CONFIG="/etc/nydus/config-guest.toml"
1620
# Toggle between true and false the service_offload configuration of
1721
# the Kata agent.
1822
#
@@ -440,3 +444,66 @@ EOF
440444
EOF
441445
fi
442446
}
447+
448+
###############################################################################
449+
450+
# remote-snapshotter
451+
452+
configure_remote_snapshotter() {
453+
case "${SNAPSHOTTER:-}" in
454+
"nydus")
455+
configure_nydus_snapshotter
456+
;;
457+
*) ;;
458+
459+
esac
460+
}
461+
check_containerd_version() {
462+
containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
463+
if echo $containerd_version | grep -q "^v1.7"; then
464+
return 1
465+
else
466+
return 0
467+
fi
468+
}
469+
configure_containerd_for_nydus_snapshotter() {
470+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG"
471+
if check_containerd_version; then
472+
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$CONTAINERD_CONFIG"
473+
else
474+
sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
475+
fi
476+
}
477+
remove_nydus_snapshotter_from_containerd() {
478+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG"
479+
if check_containerd_version; then
480+
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG"
481+
else
482+
sudo sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
483+
fi
484+
}
485+
remove_test_image() {
486+
local test_image = "$1"
487+
sudo crictl rmi "$1"
488+
pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
489+
sudo crictl rmi "$pause_name"
490+
}
491+
492+
restart_nydus_snapshotter() {
493+
echo "Kill nydus snapshotter"
494+
bin="containerd-nydus-grpc"
495+
sudo kill -9 $(pidof $bin) || true
496+
echo "Restart nydus snapshotter"
497+
sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" &
498+
}
499+
500+
configure_nydus_snapshotter() {
501+
echo "Configure nydus snapshotter"
502+
if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
503+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
504+
else
505+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
506+
fi
507+
sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
508+
restart_nydus_snapshotter
509+
}

Diff for: ‎integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
metadata:
2+
name: nydus-container$INDEX
3+
image:
4+
image: $IMAGE
5+
log_path: container.1.log

Diff for: ‎integration/kubernetes/confidential/fixtures/pod-config.yaml.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
apiVersion: v1
66
kind: Pod
77
metadata:
8-
name: busybox-cc
8+
name: busybox-cc$INDEX
99
spec:
1010
runtimeClassName: $RUNTIMECLASS
1111
containers:

Diff for: ‎integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env bats
2+
# Copyright (c) 2022 IBM Corporation
3+
#
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
7+
load "${BATS_TEST_DIRNAME}/lib.sh"
8+
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
9+
10+
# Images used on the tests.
11+
12+
image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
13+
14+
# Allow to configure the runtimeClassName on pod configuration.
15+
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
16+
test_tag="[cc][agent][kubernetes][containerd]"
17+
18+
# Create the test pod.
19+
#
20+
# Note: the global $sandbox_name, $pod_config should be set
21+
# already. It also relies on $CI and $DEBUG exported by CI scripts or
22+
# the developer, to decide how to set debug flags.
23+
#
24+
create_test_pod() {
25+
local pod_config="$1"
26+
27+
echo "Create the test sandbox"
28+
echo "Pod config is: $pod_config"
29+
crictl run --with-pull -r kata-qemu $pod_config nydus-sandbox.yaml
30+
}
31+
32+
# Create a pod configuration out of a template file.
33+
#
34+
# Parameters:
35+
# $1 - the container image.
36+
# Return:
37+
# the path to the configuration file. The caller should not care about
38+
# its removal afterwards as it is created under the bats temporary
39+
# directory.
40+
#
41+
# Environment variables:
42+
# RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS.
43+
#
44+
new_pod_config() {
45+
local base_config="${FIXTURES_DIR}/cri-pod-config.yaml.in"
46+
local image="$1"
47+
local index="$2"
48+
49+
local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
50+
IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config"
51+
echo "$new_config"
52+
}
53+
54+
setup() {
55+
start_date=$(date +"%Y-%m-%d %H:%M:%S")
56+
}
57+
58+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
59+
if [ "$SNAPSHOTTER" = "nydus" ]; then
60+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
61+
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
62+
echo $pod_config
63+
create_test_pod "$pod_config"
64+
remove_test_image "$image_unsigned_unprotected"
65+
fi
66+
}
67+
68+
@test "$test_tag Test can create two pods with pulling the image only once" {
69+
if [ "$SNAPSHOTTER" = "nydus" ]; then
70+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
71+
72+
pod_config_1="$(new_pod_config "$image_unsigned_unprotected" "1")"
73+
echo $pod_config_1
74+
create_test_pod $pod_config_1
75+
pod_config_2="$(new_pod_config "$image_unsigned_unprotected" "2")"
76+
echo $pod_config_2
77+
create_test_pod $pod_config_2
78+
79+
pull_times=$(journalctl -g "PullImage \"$image_unsigned_unprotected\" with snapshotter nydus" | wc -l)
80+
[ ${#pull_times[@]} -eq 1 ]
81+
remove_test_image "$image_unsigned_unprotected"
82+
fi
83+
}
84+
85+
@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
86+
skip
87+
switch_image_service_offload on
88+
if [ "$SNAPSHOTTER" = "nydus" ]; then
89+
EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
90+
create_test_pod
91+
remove_test_image "$image_unsigned_unprotected"
92+
fi
93+
}
94+
95+
teardown() {
96+
remove_nydus_snapshotter_from_containerd
97+
}

0 commit comments

Comments
 (0)