Skip to content

Commit d51e567

Browse files
author
ChengyuZhu6
committed
ci: Add test cases for CoCo image pulling without forked containerd
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. 1)image sharing on the host without dm-verity. 2)image sharing on the host with dm-verity. 3)two pods with pulling image only once. 4)image pulling in the guest with nydus-snapshotter. Fixes kata-containers#5763 Depends-on: github.com/kata-containers/kata-containers#7676 Signed-off-by: ChengyuZhu6 <[email protected]>
1 parent 6d7723a commit d51e567

12 files changed

+292
-30
lines changed

Diff for: ‎.ci/install_nydus_snapshotter.sh

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/bin/bash
2+
#
3+
# Copyright (c) 2023 Intel Corporation
4+
#
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
8+
set -o errexit
9+
set -o nounset
10+
set -o pipefail
11+
set -o errtrace
12+
13+
cidir=$(dirname "$0")
14+
source "${cidir}/lib.sh"
15+
16+
target_dir="/opt/confidential-containers"
17+
18+
nydus_snapshotter_repo=${nydus_snapshotter_repo:-"github.com/containerd/nydus-snapshotter"}
19+
nydus_snapshotter_repo_git="https://${nydus_snapshotter_repo}.git"
20+
nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.12.0"}
21+
nydus_snapshotter_repo_dir="${GOPATH}/src/${nydus_snapshotter_repo}"
22+
nydus_snapshotter_binary_target_dir="$target_dir/bin"
23+
nydus_snapshotter_config_target_dir="$target_dir/share/nydus-snapshotter"
24+
25+
nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"}
26+
nydus_version=${nydus_version:-"v2.2.3"}
27+
28+
arch="$(uname -m)"
29+
30+
clone_nydus_snapshotter_repo() {
31+
add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}"
32+
33+
if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then
34+
sudo mkdir -p "${nydus_snapshotter_repo_dir}"
35+
git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true
36+
pushd "${nydus_snapshotter_repo_dir}"
37+
git checkout "${nydus_snapshotter_version}"
38+
popd
39+
fi
40+
}
41+
42+
build_nydus_snapshotter() {
43+
pushd "${nydus_snapshotter_repo_dir}"
44+
if [ "$arch" = "s390x" ]; then
45+
export GOARCH=$arch
46+
fi
47+
sudo -E PATH=$PATH make
48+
49+
if [ ! -d "$nydus_snapshotter_binary_target_dir" ]; then
50+
sudo mkdir -p $nydus_snapshotter_binary_target_dir
51+
fi
52+
sudo install -D -m 755 "bin/containerd-nydus-grpc" "$nydus_snapshotter_binary_target_dir/containerd-nydus-grpc"
53+
sudo install -D -m 755 "bin/nydus-overlayfs" "$nydus_snapshotter_binary_target_dir/nydus-overlayfs"
54+
if [ ! -f "/usr/local/bin/nydus-overlayfs" ]; then
55+
echo " /usr/local/bin/nydus-overlayfs exists, now we will replace it."
56+
sudo cp -f "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs"
57+
fi
58+
sudo rm -rf "$nydus_snapshotter_repo_dir/bin"
59+
popd >/dev/null
60+
}
61+
62+
download_nydus_snapshotter_config() {
63+
if [ ! -d "$nydus_snapshotter_config_target_dir" ]; then
64+
mkdir -p "$nydus_snapshotter_config_target_dir"
65+
fi
66+
sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
67+
sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
68+
sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
69+
sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
70+
71+
}
72+
73+
download_nydus_from_tarball() {
74+
if [ "$arch" = "s390x" ]; then
75+
echo "Skip to download nydus for $arch, it doesn't work for $arch now."
76+
return
77+
fi
78+
local goarch="$(${cidir}/kata-arch.sh --golang)"
79+
local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-$goarch.tgz"
80+
echo "Download tarball from ${tarball_url}"
81+
tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX)
82+
curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1
83+
sudo install -D -m 755 "$tmp_dir/nydus-image" "/usr/local/bin/"
84+
}
85+
86+
download_nydus_from_tarball
87+
clone_nydus_snapshotter_repo
88+
build_nydus_snapshotter
89+
download_nydus_snapshotter_config
90+
echo "install nydus-snapshotter successful"

Diff for: ‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ cc-containerd:
104104
# Run the Confidential Containers tests for kubernetes.
105105
cc-kubernetes:
106106
bash -f .ci/install_bats.sh
107-
K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \
107+
K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \
108108
bash integration/kubernetes/run_kubernetes_tests.sh
109109

110110
# Run the Confidential Containers AMD SEV specific tests.

Diff for: ‎integration/confidential/lib.sh

+74-7
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
1212
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
1313
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
1414
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
15+
NYDUS_SNAPSHOTTER_BINARY="/opt/kata/bin/containerd-nydus-grpc"
16+
NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-host-sharing.toml"
17+
NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-guest-pulling.toml"
18+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
1519

1620
# Toggle between true and false the service_offload configuration of
1721
# the Kata agent.
@@ -180,6 +184,13 @@ disable_full_debug() {
180184
sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH"
181185
}
182186

187+
restart_containerd(){
188+
sudo systemctl restart containerd
189+
if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
190+
die "containerd seems not operational after restarted"
191+
fi
192+
}
193+
183194
# Configure containerd for confidential containers. Among other things, it ensures
184195
# the CRI handler is configured to deal with confidential container.
185196
#
@@ -198,15 +209,15 @@ configure_cc_containerd() {
198209
# installed via operator it will assume containerd is in right state
199210
# already.
200211
[ "${TESTS_CONFIGURE_CC_CONTAINERD:-yes}" == "yes" ] || return 0
212+
sudo iptables -w -P FORWARD ACCEPT
201213

202214
# Even if we are not saving the original file it is a good idea to
203215
# restart containerd because it might be in an inconsistent state here.
204216
sudo systemctl stop containerd
205217
sleep 5
206218
[ -n "$saved_containerd_conf_file" ] && \
207219
sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
208-
sudo systemctl start containerd
209-
waitForProcess 30 5 "sudo crictl info >/dev/null"
220+
restart_containerd
210221

211222
# Ensure the cc CRI handler is set.
212223
local cri_handler=$(sudo crictl info | \
@@ -223,11 +234,6 @@ configure_cc_containerd() {
223234
sudo tee -a "$containerd_conf_file"
224235
fi
225236

226-
sudo systemctl restart containerd
227-
if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
228-
die "containerd seems not operational after reconfigured"
229-
fi
230-
sudo iptables -w -P FORWARD ACCEPT
231237
}
232238

233239
#
@@ -445,3 +451,64 @@ EOF
445451
EOF
446452
fi
447453
}
454+
455+
###############################################################################
456+
457+
# remote-snapshotter
458+
459+
configure_remote_snapshotter() {
460+
case "${SNAPSHOTTER:-}" in
461+
"nydus")
462+
configure_nydus_snapshotter
463+
;;
464+
*) ;;
465+
466+
esac
467+
}
468+
469+
is_containerd_support_per_runtime_snapshotter () {
470+
containerd_version=$(containerd --version | awk '{print $3}')
471+
required_version="v1.7.0"
472+
printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet -V
473+
}
474+
475+
configure_containerd_for_nydus_snapshotter() {
476+
if [ "${SNAPSHOTTER:-}" = "nydus" ]; then
477+
local containerd_config="$1"
478+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config"
479+
sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config"
480+
fi
481+
}
482+
483+
kill_nydus_snapshotter_process() {
484+
echo "Kill nydus snapshotter"
485+
bin="containerd-nydus-grpc"
486+
sudo kill -9 $(pidof $bin) || true
487+
sudo rm -rf "/var/lib/containerd-nydus" || true
488+
}
489+
490+
remove_test_image() {
491+
local test_image="$1"
492+
crictl rmi "$1"
493+
pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
494+
crictl rmi "$pause_name"
495+
}
496+
497+
restart_nydus_snapshotter() {
498+
kill_nydus_snapshotter_process || true
499+
echo "Restart nydus snapshotter"
500+
sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 &
501+
}
502+
503+
configure_nydus_snapshotter() {
504+
if [ "${SNAPSHOTTER:-}" = "nydus" ]; then
505+
echo "Configure nydus snapshotter"
506+
if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
507+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
508+
else
509+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
510+
sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
511+
fi
512+
restart_nydus_snapshotter
513+
fi
514+
}

Diff for: ‎integration/kubernetes/confidential/agent_image.bats

+5-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
# SPDX-License-Identifier: Apache-2.0
55
#
66

7+
load "${BATS_TEST_DIRNAME}/lib.sh"
8+
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
79
load "${BATS_TEST_DIRNAME}/tests_common.sh"
810

911
tag_suffix=""
@@ -31,7 +33,9 @@ RUNTIMECLASS="${RUNTIMECLASS:-kata}"
3133
test_tag="[cc][agent][kubernetes][containerd]"
3234

3335
setup() {
34-
setup_common
36+
setup_containerd
37+
restart_containerd
38+
reconfigure_kata
3539
}
3640

3741
@test "$test_tag Test can launch pod with measured boot enabled" {

Diff for: ‎integration/kubernetes/confidential/agent_image_encrypted.bats

+2-7
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,8 @@ setup() {
2525
SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
2626
configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
2727

28-
echo "Reconfigure Kata Containers"
29-
switch_image_service_offload on
30-
clear_kernel_params
31-
add_kernel_params "${original_kernel_params}"
32-
33-
setup_proxy
34-
switch_measured_rootfs_verity_scheme none
28+
restart_containerd
29+
reconfigure_kata
3530
}
3631

3732
@test "$test_tag Test can pull an encrypted image inside the guest with decryption key" {

Diff for: ‎integration/kubernetes/confidential/fixtures/pod-config.yaml.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
apiVersion: v1
66
kind: Pod
77
metadata:
8-
name: busybox-cc
8+
name: busybox-cc$INDEX
99
spec:
1010
runtimeClassName: $RUNTIMECLASS
1111
containers:

Diff for: ‎integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/usr/bin/env bats
2+
# Copyright (c) 2023 Intel Corporation
3+
#
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
7+
load "${BATS_TEST_DIRNAME}/lib.sh"
8+
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
9+
load "${BATS_TEST_DIRNAME}/tests_common.sh"
10+
11+
tag_suffix=""
12+
if [ "$(uname -m)" != "x86_64" ]; then
13+
tag_suffix="-$(uname -m)"
14+
fi
15+
16+
# Images used on the tests.
17+
18+
image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
19+
20+
original_kernel_params=$(get_kernel_params)
21+
# Allow to configure the runtimeClassName on pod configuration.
22+
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
23+
test_tag="[cc][agent][kubernetes][containerd]"
24+
25+
setup() {
26+
remove_test_image "$image_unsigned_protected" || true
27+
setup_containerd
28+
configure_containerd_for_nydus_snapshotter "$containerd_conf_file"
29+
restart_containerd
30+
reconfigure_kata
31+
}
32+
33+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
34+
if [ "$(uname -m)" = s390x ]; then
35+
skip "test for s390x as nydus-image doesn't currently support this platform"
36+
fi
37+
if [ "$SNAPSHOTTER" = "nydus" ]; then
38+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
39+
pod_config="$(new_pod_config "$image_unsigned_protected")"
40+
echo $pod_config
41+
create_test_pod
42+
fi
43+
}
44+
45+
@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" {
46+
if [ "$(uname -m)" = s390x ]; then
47+
skip "test for s390x as nydus-image doesn't currently support this platform"
48+
fi
49+
if [ "$SNAPSHOTTER" = "nydus" ]; then
50+
EXPORT_MODE="image_block" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
51+
pod_config="$(new_pod_config "$image_unsigned_protected")"
52+
echo $pod_config
53+
create_test_pod
54+
fi
55+
}
56+
57+
@test "$test_tag Test can create two pods with pulling the image only once with dm-verity enabled" {
58+
if [ "$(uname -m)" = s390x ]; then
59+
skip "test for s390x as nydus-image doesn't currently support this platform"
60+
fi
61+
if [ "$SNAPSHOTTER" = "nydus" ]; then
62+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
63+
64+
pod_config="$(new_pod_config "$image_unsigned_protected" "1")"
65+
echo $pod_config
66+
create_test_pod
67+
pod_config="$(new_pod_config "$image_unsigned_protected" "2")"
68+
echo $pod_config
69+
create_test_pod
70+
71+
pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l)
72+
[ ${pull_times} -eq 1 ]
73+
fi
74+
}
75+
76+
@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
77+
switch_image_service_offload on
78+
if [ "$SNAPSHOTTER" = "nydus" ]; then
79+
EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
80+
pod_config="$(new_pod_config "$image_unsigned_protected")"
81+
echo $pod_config
82+
create_test_pod
83+
fi
84+
}
85+
86+
teardown() {
87+
teardown_common
88+
remove_test_image "$image_unsigned_protected" || true
89+
kill_nydus_snapshotter_process
90+
}

Diff for: ‎integration/kubernetes/confidential/lib.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ kubernetes_create_cc_pod() {
4646
fi
4747

4848
kubectl apply -f ${config_file}
49-
if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
49+
pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name')
50+
if ! kubectl get pod "$pod_name" &> /dev/null; then
5051
echo "Failed to create the pod"
5152
return 1
5253
fi

Diff for: ‎integration/kubernetes/confidential/sealed_secret.bats

+2-7
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,8 @@ setup() {
2222
SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
2323
configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
2424

25-
echo "Reconfigure Kata Containers"
26-
switch_image_service_offload on
27-
clear_kernel_params
28-
add_kernel_params "${original_kernel_params}"
29-
30-
setup_proxy
31-
switch_measured_rootfs_verity_scheme none
25+
restart_containerd
26+
reconfigure_kata
3227

3328
kubectl delete secret sealed-secret --ignore-not-found
3429
# Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault

Diff for: ‎integration/kubernetes/confidential/sev.bats

+2-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ setup_file() {
5555
# Configure CoCo settings in containerd config
5656
local saved_containerd_conf_file="/etc/containerd/config.toml.$$"
5757
configure_cc_containerd "${saved_containerd_conf_file}"
58-
58+
restart_containerd
59+
5960
# KBS setup and run
6061
echo "Setting up simple-kbs..."
6162
simple_kbs_run

0 commit comments

Comments
 (0)