Skip to content

Commit 0482506

Browse files
author
ChengyuZhu6
committed
ci: Add test cases for CoCo image pulling without forked containerd
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. 1)image sharing on the host without dm-verity. 2)image sharing on the host with dm-verity. 3)two pods with pulling image only once. 4)image pulling in the guest with nydus-snapshotter. Fixes kata-containers#5763 Depends-on: github.com/kata-containers/kata-containers#7676 Signed-off-by: ChengyuZhu6 <[email protected]>
1 parent 6d7723a commit 0482506

File tree

13 files changed

+298
-31
lines changed

13 files changed

+298
-31
lines changed

Diff for: ‎.ci/install_nydus_snapshotter.sh

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/bin/bash
2+
#
3+
# Copyright (c) 2023 Intel Corporation
4+
#
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
8+
set -o errexit
9+
set -o nounset
10+
set -o pipefail
11+
set -o errtrace
12+
13+
cidir=$(dirname "$0")
14+
source "${cidir}/lib.sh"
15+
16+
target_dir="/opt/confidential-containers"
17+
18+
nydus_snapshotter_repo=${nydus_snapshotter_repo:-"github.com/containerd/nydus-snapshotter"}
19+
nydus_snapshotter_repo_git="https://${nydus_snapshotter_repo}.git"
20+
nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.12.0"}
21+
nydus_snapshotter_repo_dir="${GOPATH}/src/${nydus_snapshotter_repo}"
22+
nydus_snapshotter_binary_target_dir="$target_dir/bin"
23+
nydus_snapshotter_config_target_dir="$target_dir/share/nydus-snapshotter"
24+
25+
nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"}
26+
nydus_version=${nydus_version:-"v2.2.3"}
27+
28+
arch="$(uname -m)"
29+
30+
clone_nydus_snapshotter_repo() {
31+
add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}"
32+
33+
if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then
34+
sudo mkdir -p "${nydus_snapshotter_repo_dir}"
35+
git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true
36+
pushd "${nydus_snapshotter_repo_dir}"
37+
git checkout "${nydus_snapshotter_version}"
38+
popd
39+
fi
40+
}
41+
42+
build_nydus_snapshotter() {
43+
pushd "${nydus_snapshotter_repo_dir}"
44+
if [ "$arch" = "s390x" ]; then
45+
export GOARCH=$arch
46+
fi
47+
sudo -E PATH=$PATH make
48+
49+
if [ ! -d "$nydus_snapshotter_binary_target_dir" ]; then
50+
sudo mkdir -p $nydus_snapshotter_binary_target_dir
51+
fi
52+
sudo install -D -m 755 "bin/containerd-nydus-grpc" "$nydus_snapshotter_binary_target_dir/containerd-nydus-grpc"
53+
sudo install -D -m 755 "bin/nydus-overlayfs" "$nydus_snapshotter_binary_target_dir/nydus-overlayfs"
54+
if [ ! -f "/usr/local/bin/nydus-overlayfs" ]; then
55+
echo " /usr/local/bin/nydus-overlayfs exists, now we will replace it."
56+
sudo cp -f "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs"
57+
fi
58+
sudo rm -rf "$nydus_snapshotter_repo_dir/bin"
59+
popd >/dev/null
60+
}
61+
62+
download_nydus_snapshotter_config() {
63+
if [ ! -d "$nydus_snapshotter_config_target_dir" ]; then
64+
mkdir -p "$nydus_snapshotter_config_target_dir"
65+
fi
66+
sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
67+
sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
68+
sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
69+
sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
70+
71+
}
72+
73+
download_nydus_from_tarball() {
74+
if [ "$arch" = "s390x" ]; then
75+
echo "Skip to download nydus for $arch, it doesn't work for $arch now."
76+
return
77+
fi
78+
local goarch="$(${cidir}/kata-arch.sh --golang)"
79+
local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-$goarch.tgz"
80+
echo "Download tarball from ${tarball_url}"
81+
tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX)
82+
curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1
83+
sudo install -D -m 755 "$tmp_dir/nydus-image" "/usr/local/bin/"
84+
}
85+
86+
download_nydus_from_tarball
87+
clone_nydus_snapshotter_repo
88+
build_nydus_snapshotter
89+
download_nydus_snapshotter_config
90+
echo "install nydus-snapshotter successful"

Diff for: ‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ cc-containerd:
104104
# Run the Confidential Containers tests for kubernetes.
105105
cc-kubernetes:
106106
bash -f .ci/install_bats.sh
107-
K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \
107+
K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \
108108
bash integration/kubernetes/run_kubernetes_tests.sh
109109

110110
# Run the Confidential Containers AMD SEV specific tests.

Diff for: ‎integration/confidential/lib.sh

+74-7
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
1212
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
1313
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
1414
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
15+
NYDUS_SNAPSHOTTER_BINARY="/opt/kata/bin/containerd-nydus-grpc"
16+
NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-host-sharing.toml"
17+
NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-guest-pulling.toml"
18+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
1519

1620
# Toggle between true and false the service_offload configuration of
1721
# the Kata agent.
@@ -180,6 +184,13 @@ disable_full_debug() {
180184
sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH"
181185
}
182186

187+
restart_containerd(){
188+
sudo systemctl restart containerd
189+
if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
190+
die "containerd seems not operational after restarted"
191+
fi
192+
}
193+
183194
# Configure containerd for confidential containers. Among other things, it ensures
184195
# the CRI handler is configured to deal with confidential container.
185196
#
@@ -198,15 +209,15 @@ configure_cc_containerd() {
198209
# installed via operator it will assume containerd is in right state
199210
# already.
200211
[ "${TESTS_CONFIGURE_CC_CONTAINERD:-yes}" == "yes" ] || return 0
212+
sudo iptables -w -P FORWARD ACCEPT
201213

202214
# Even if we are not saving the original file it is a good idea to
203215
# restart containerd because it might be in an inconsistent state here.
204216
sudo systemctl stop containerd
205217
sleep 5
206218
[ -n "$saved_containerd_conf_file" ] && \
207219
sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
208-
sudo systemctl start containerd
209-
waitForProcess 30 5 "sudo crictl info >/dev/null"
220+
restart_containerd
210221

211222
# Ensure the cc CRI handler is set.
212223
local cri_handler=$(sudo crictl info | \
@@ -223,11 +234,6 @@ configure_cc_containerd() {
223234
sudo tee -a "$containerd_conf_file"
224235
fi
225236

226-
sudo systemctl restart containerd
227-
if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
228-
die "containerd seems not operational after reconfigured"
229-
fi
230-
sudo iptables -w -P FORWARD ACCEPT
231237
}
232238

233239
#
@@ -445,3 +451,64 @@ EOF
445451
EOF
446452
fi
447453
}
454+
455+
###############################################################################
456+
457+
# remote-snapshotter
458+
459+
configure_remote_snapshotter() {
460+
case "${SNAPSHOTTER:-}" in
461+
"nydus")
462+
configure_nydus_snapshotter
463+
;;
464+
*) ;;
465+
466+
esac
467+
}
468+
469+
is_containerd_support_per_runtime_snapshotter () {
470+
containerd_version=$(containerd --version | awk '{print $3}')
471+
required_version="v1.7.0"
472+
printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet -V
473+
}
474+
475+
configure_containerd_for_nydus_snapshotter() {
476+
if [ "${SNAPSHOTTER:-}" = "nydus" ]; then
477+
local containerd_config="$1"
478+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config"
479+
sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config"
480+
fi
481+
}
482+
483+
kill_nydus_snapshotter_process() {
484+
echo "Kill nydus snapshotter"
485+
bin="containerd-nydus-grpc"
486+
sudo kill -9 $(pidof $bin) || true
487+
sudo rm -rf "/var/lib/containerd-nydus" || true
488+
}
489+
490+
remove_test_image() {
491+
local test_image="$1"
492+
crictl rmi "$1"
493+
pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
494+
crictl rmi "$pause_name"
495+
}
496+
497+
restart_nydus_snapshotter() {
498+
kill_nydus_snapshotter_process || true
499+
echo "Restart nydus snapshotter"
500+
sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 &
501+
}
502+
503+
configure_nydus_snapshotter() {
504+
if [ "${SNAPSHOTTER:-}" = "nydus" ]; then
505+
echo "Configure nydus snapshotter"
506+
if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
507+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
508+
else
509+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
510+
sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
511+
fi
512+
restart_nydus_snapshotter
513+
fi
514+
}

Diff for: ‎integration/containerd/confidential/tests_common.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ setup_common() {
2626
echo "Prepare containerd for Confidential Container"
2727
SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
2828
configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
29-
29+
restart_containerd
3030
# Note: ensure that intructions changing the kernel parameters are
3131
# executed *after* saving the original list.
3232
saved_kernel_params=$(get_kernel_params)

Diff for: ‎integration/kubernetes/confidential/agent_image.bats

+5-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
# SPDX-License-Identifier: Apache-2.0
55
#
66

7+
load "${BATS_TEST_DIRNAME}/lib.sh"
8+
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
79
load "${BATS_TEST_DIRNAME}/tests_common.sh"
810

911
tag_suffix=""
@@ -31,7 +33,9 @@ RUNTIMECLASS="${RUNTIMECLASS:-kata}"
3133
test_tag="[cc][agent][kubernetes][containerd]"
3234

3335
setup() {
34-
setup_common
36+
setup_containerd
37+
restart_containerd
38+
reconfigure_kata
3539
}
3640

3741
@test "$test_tag Test can launch pod with measured boot enabled" {

Diff for: ‎integration/kubernetes/confidential/agent_image_encrypted.bats

+3-7
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
load "${BATS_TEST_DIRNAME}/lib.sh"
88
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
99
load "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
10+
load "${BATS_TEST_DIRNAME}/tests_common.sh"
1011

1112
# Allow to configure the runtimeClassName on pod configuration.
1213
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
@@ -25,13 +26,8 @@ setup() {
2526
SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
2627
configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
2728

28-
echo "Reconfigure Kata Containers"
29-
switch_image_service_offload on
30-
clear_kernel_params
31-
add_kernel_params "${original_kernel_params}"
32-
33-
setup_proxy
34-
switch_measured_rootfs_verity_scheme none
29+
restart_containerd
30+
reconfigure_kata
3531
}
3632

3733
@test "$test_tag Test can pull an encrypted image inside the guest with decryption key" {

Diff for: ‎integration/kubernetes/confidential/fixtures/pod-config.yaml.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
apiVersion: v1
66
kind: Pod
77
metadata:
8-
name: busybox-cc
8+
name: busybox-cc$INDEX
99
spec:
1010
runtimeClassName: $RUNTIMECLASS
1111
containers:

Diff for: ‎integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/usr/bin/env bats
2+
# Copyright (c) 2023 Intel Corporation
3+
#
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
7+
load "${BATS_TEST_DIRNAME}/lib.sh"
8+
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
9+
load "${BATS_TEST_DIRNAME}/tests_common.sh"
10+
11+
tag_suffix=""
12+
if [ "$(uname -m)" != "x86_64" ]; then
13+
tag_suffix="-$(uname -m)"
14+
fi
15+
16+
# Images used on the tests.
17+
18+
image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
19+
20+
original_kernel_params=$(get_kernel_params)
21+
# Allow to configure the runtimeClassName on pod configuration.
22+
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
23+
test_tag="[cc][agent][kubernetes][containerd]"
24+
25+
setup() {
26+
remove_test_image "$image_unsigned_protected" || true
27+
setup_containerd
28+
configure_containerd_for_nydus_snapshotter "/etc/containerd/config.toml"
29+
restart_containerd
30+
reconfigure_kata
31+
switch_image_service_offload off
32+
}
33+
34+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
35+
if [ "$(uname -m)" = s390x ]; then
36+
skip "test for s390x as nydus-image doesn't currently support this platform"
37+
fi
38+
if [ "$SNAPSHOTTER" = "nydus" ]; then
39+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
40+
pod_config="$(new_pod_config "$image_unsigned_protected")"
41+
echo $pod_config
42+
create_test_pod
43+
fi
44+
}
45+
46+
@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" {
47+
if [ "$(uname -m)" = s390x ]; then
48+
skip "test for s390x as nydus-image doesn't currently support this platform"
49+
fi
50+
if [ "$SNAPSHOTTER" = "nydus" ]; then
51+
EXPORT_MODE="image_block" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
52+
pod_config="$(new_pod_config "$image_unsigned_protected")"
53+
echo $pod_config
54+
create_test_pod
55+
fi
56+
}
57+
58+
@test "$test_tag Test can create two pods with pulling the image only once with dm-verity enabled" {
59+
if [ "$(uname -m)" = s390x ]; then
60+
skip "test for s390x as nydus-image doesn't currently support this platform"
61+
fi
62+
if [ "$SNAPSHOTTER" = "nydus" ]; then
63+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
64+
65+
pod_config="$(new_pod_config "$image_unsigned_protected" "1")"
66+
echo $pod_config
67+
create_test_pod
68+
pod_config="$(new_pod_config "$image_unsigned_protected" "2")"
69+
echo $pod_config
70+
create_test_pod
71+
72+
pull_times=$(journalctl -t containerd -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l)
73+
[ ${pull_times} -eq 1 ]
74+
fi
75+
}
76+
77+
@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
78+
switch_image_service_offload on
79+
if [ "$SNAPSHOTTER" = "nydus" ]; then
80+
EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
81+
pod_config="$(new_pod_config "$image_unsigned_protected")"
82+
echo $pod_config
83+
create_test_pod
84+
fi
85+
}
86+
87+
teardown() {
88+
teardown_common
89+
remove_test_image "$image_unsigned_protected" || true
90+
kill_nydus_snapshotter_process
91+
}

Diff for: ‎integration/kubernetes/confidential/lib.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ kubernetes_create_cc_pod() {
4646
fi
4747

4848
kubectl apply -f ${config_file}
49-
if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
49+
pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name')
50+
if ! kubectl get pod "$pod_name" &> /dev/null; then
5051
echo "Failed to create the pod"
5152
return 1
5253
fi

0 commit comments

Comments
 (0)