Skip to content

Commit 6fb1af5

Browse files
author
ChengyuZhu6
committed
ci: Add test cases for CoCo image pulling without forked containerd
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. Fixes kata-containers#5763 Depends: kata-containers/kata-containers#7688 kata-containers/kata-containers#7676 Signed-off-by: ChengyuZhu6 <[email protected]>
1 parent 021bf58 commit 6fb1af5

File tree

4 files changed

+179
-2
lines changed

4 files changed

+179
-2
lines changed

Diff for: ‎integration/confidential/lib.sh

+68-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
1212
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
1313
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
1414
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
15-
15+
CONTAINERD_CONFIG="/etc/containerd/config.toml"
16+
NYDUS_SNAPSHOTTER_BINARY="/home/zcy/workspace/image_sharing/bin/containerd-nydus-grpc"
17+
NYDUS_SNAPSHOTTER_CONFIG="/etc/nydus/config-tarfs.toml"
18+
NYDUS_SNAPSHOTTER_TARFS_CONFIG="/etc/nydus/config-tarfs.toml"
19+
NYDUS_SNAPSHOTTER_GUEST_CONFIG="/etc/nydus/config-guest.toml"
1620
# Toggle between true and false the service_offload configuration of
1721
# the Kata agent.
1822
#
@@ -440,3 +444,66 @@ EOF
440444
EOF
441445
fi
442446
}
447+
448+
###############################################################################
449+
450+
# remote-snapshotter
451+
452+
configure_remote_snapshotter() {
453+
case "${SNAPSHOTTER:-}" in
454+
"nydus")
455+
configure_nydus_snapshotter
456+
;;
457+
*) ;;
458+
459+
esac
460+
}
461+
check_containerd_version() {
462+
containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
463+
if echo $containerd_version | grep -q "^v1.7"; then
464+
return 1
465+
else
466+
return 0
467+
fi
468+
}
469+
configure_containerd_for_nydus_snapshotter() {
470+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG"
471+
if check_containerd_version; then
472+
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$CONTAINERD_CONFIG"
473+
else
474+
sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
475+
fi
476+
}
477+
remove_nydus_snapshotter_from_containerd() {
478+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG"
479+
if check_containerd_version; then
480+
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG"
481+
else
482+
sudo sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
483+
fi
484+
}
485+
remove_test_image() {
486+
local test_image = "$1"
487+
sudo crictl rmi "$1"
488+
pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
489+
sudo crictl rmi "$pause_name"
490+
}
491+
492+
restart_nydus_snapshotter() {
493+
echo "Kill nydus snapshotter"
494+
bin="containerd-nydus-grpc"
495+
sudo kill -9 $(pidof $bin) || true
496+
echo "Restart nydus snapshotter"
497+
sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" &
498+
}
499+
500+
configure_nydus_snapshotter() {
501+
echo "Configure nydus snapshotter"
502+
if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
503+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
504+
else
505+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
506+
fi
507+
sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
508+
restart_nydus_snapshotter
509+
}

Diff for: ‎integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
metadata:
2+
name: nydus-container$INDEX
3+
image:
4+
image: $IMAGE
5+
log_path: container.1.log

Diff for: ‎integration/kubernetes/confidential/fixtures/pod-config.yaml.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
apiVersion: v1
66
kind: Pod
77
metadata:
8-
name: busybox-cc
8+
name: busybox-cc$INDEX
99
spec:
1010
runtimeClassName: $RUNTIMECLASS
1111
containers:

Diff for: ‎integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/usr/bin/env bats
2+
# Copyright (c) 2022 IBM Corporation
3+
#
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
7+
load "${BATS_TEST_DIRNAME}/lib.sh"
8+
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
9+
10+
tag_suffix=""
11+
if [ "$(uname -m)" != "x86_64" ]; then
12+
tag_suffix="-$(uname -m)"
13+
fi
14+
15+
# Images used on the tests.
16+
17+
image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
18+
19+
original_kernel_params=$(get_kernel_params)
20+
# Allow to configure the runtimeClassName on pod configuration.
21+
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
22+
test_tag="[cc][agent][kubernetes][containerd]"
23+
24+
# Create the test pod.
25+
#
26+
# Note: the global $sandbox_name, $pod_config should be set
27+
# already. It also relies on $CI and $DEBUG exported by CI scripts or
28+
# the developer, to decide how to set debug flags.
29+
#
30+
create_test_pod() {
31+
local pod_config="$1"
32+
33+
echo "Create the test sandbox"
34+
echo "Pod config is: $pod_config"
35+
crictl crictl run --with-pull -r kata-qemu $pod_config nydus-sandbox.yaml
36+
}
37+
38+
# Create a pod configuration out of a template file.
39+
#
40+
# Parameters:
41+
# $1 - the container image.
42+
# Return:
43+
# the path to the configuration file. The caller should not care about
44+
# its removal afterwards as it is created under the bats temporary
45+
# directory.
46+
#
47+
# Environment variables:
48+
# RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS.
49+
#
50+
new_pod_config() {
51+
local base_config="${FIXTURES_DIR}/cri-pod-config.yaml.in"
52+
local image="$1"
53+
local index="$2"
54+
55+
local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
56+
IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config"
57+
echo "$new_config"
58+
}
59+
60+
setup() {
61+
start_date=$(date +"%Y-%m-%d %H:%M:%S")
62+
setup_proxy
63+
switch_measured_rootfs_verity_scheme none
64+
}
65+
66+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
67+
if [ "$SNAPSHOTTER" = "nydus" ]; then
68+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
69+
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
70+
echo $pod_config
71+
create_test_pod "$pod_config"
72+
remove_test_image "$image_unsigned_unprotected"
73+
fi
74+
}
75+
76+
@test "$test_tag Test can create two pods with pulling the image only once" {
77+
if [ "$SNAPSHOTTER" = "nydus" ]; then
78+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
79+
80+
pod_config_1="$(new_pod_config "$image_unsigned_unprotected" "1")"
81+
echo $pod_config_1
82+
create_test_pod $pod_config_1
83+
pod_config_2="$(new_pod_config "$image_unsigned_unprotected" "2")"
84+
echo $pod_config_2
85+
create_test_pod $pod_config_2
86+
87+
pull_times=$(journalctl -g "PullImage \"$image_unsigned_unprotected\" with snapshotter nydus" | wc -l)
88+
[ ${#pull_times[@]} -eq 1 ]
89+
remove_test_image "$image_unsigned_unprotected"
90+
fi
91+
}
92+
93+
@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
94+
skip
95+
switch_image_service_offload on
96+
if [ "$SNAPSHOTTER" = "nydus" ]; then
97+
EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
98+
create_test_pod
99+
remove_test_image "$image_unsigned_unprotected"
100+
fi
101+
}
102+
103+
teardown() {
104+
remove_nydus_snapshotter_from_containerd
105+
}

0 commit comments

Comments
 (0)