Skip to content

Commit 6de25d9

Browse files
author
ChengyuZhu6
committed
ci: Add test cases for CoCo image pulling without forked containerd
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. 1)image sharing on the host without dm-verity. 2)image sharing on the host with dm-verity. 3)image pulling in the guest with nydus-snapshotter. Fixes kata-containers#5763 Depends-on: github.com/kata-containers/kata-containers#7676 Signed-off-by: ChengyuZhu6 <[email protected]>
1 parent 6cf4306 commit 6de25d9

File tree

13 files changed

+302
-33
lines changed

13 files changed

+302
-33
lines changed

Diff for: ‎.ci/install_nydus_snapshotter.sh

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/bin/bash
2+
#
3+
# Copyright (c) 2023 Intel Corporation
4+
#
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
8+
set -o errexit
9+
set -o nounset
10+
set -o pipefail
11+
set -o errtrace
12+
13+
cidir=$(dirname "$0")
14+
source "${cidir}/lib.sh"
15+
16+
target_dir="/opt/confidential-containers"
17+
18+
nydus_snapshotter_repo=${nydus_snapshotter_repo:-"github.com/containerd/nydus-snapshotter"}
19+
nydus_snapshotter_repo_git="https://${nydus_snapshotter_repo}.git"
20+
nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.12.0"}
21+
nydus_snapshotter_repo_dir="${GOPATH}/src/${nydus_snapshotter_repo}"
22+
nydus_snapshotter_binary_target_dir="$target_dir/bin"
23+
nydus_snapshotter_config_target_dir="$target_dir/share/nydus-snapshotter"
24+
25+
nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"}
26+
nydus_version=${nydus_version:-"v2.2.3"}
27+
28+
arch="$(uname -m)"
29+
30+
clone_nydus_snapshotter_repo() {
31+
add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}"
32+
33+
if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then
34+
sudo mkdir -p "${nydus_snapshotter_repo_dir}"
35+
git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true
36+
pushd "${nydus_snapshotter_repo_dir}"
37+
git checkout "${nydus_snapshotter_version}"
38+
popd
39+
fi
40+
}
41+
42+
build_nydus_snapshotter() {
43+
pushd "${nydus_snapshotter_repo_dir}"
44+
if [ "$arch" = "s390x" ]; then
45+
export GOARCH=$arch
46+
fi
47+
sudo -E PATH=$PATH make
48+
49+
if [ ! -d "$nydus_snapshotter_binary_target_dir" ]; then
50+
sudo mkdir -p $nydus_snapshotter_binary_target_dir
51+
fi
52+
sudo install -D -m 755 "bin/containerd-nydus-grpc" "$nydus_snapshotter_binary_target_dir/containerd-nydus-grpc"
53+
sudo install -D -m 755 "bin/nydus-overlayfs" "$nydus_snapshotter_binary_target_dir/nydus-overlayfs"
54+
if [ ! -f "/usr/local/bin/nydus-overlayfs" ]; then
55+
echo " /usr/local/bin/nydus-overlayfs exists, now we will replace it."
56+
sudo cp -f "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs"
57+
fi
58+
sudo rm -rf "$nydus_snapshotter_repo_dir/bin"
59+
popd >/dev/null
60+
}
61+
62+
download_nydus_snapshotter_config() {
63+
if [ ! -d "$nydus_snapshotter_config_target_dir" ]; then
64+
mkdir -p "$nydus_snapshotter_config_target_dir"
65+
fi
66+
sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
67+
sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
68+
sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml"
69+
sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml"
70+
71+
}
72+
73+
download_nydus_from_tarball() {
74+
if [ "$arch" = "s390x" ]; then
75+
echo "Skip to download nydus for $arch, it doesn't work for $arch now."
76+
return
77+
fi
78+
local goarch="$(${cidir}/kata-arch.sh --golang)"
79+
local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-$goarch.tgz"
80+
echo "Download tarball from ${tarball_url}"
81+
tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX)
82+
curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1
83+
sudo install -D -m 755 "$tmp_dir/nydus-image" "/usr/local/bin/"
84+
}
85+
86+
download_nydus_from_tarball
87+
clone_nydus_snapshotter_repo
88+
build_nydus_snapshotter
89+
download_nydus_snapshotter_config
90+
echo "install nydus-snapshotter successful"

Diff for: ‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ cc-containerd:
104104
# Run the Confidential Containers tests for kubernetes.
105105
cc-kubernetes:
106106
bash -f .ci/install_bats.sh
107-
K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \
107+
K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \
108108
bash integration/kubernetes/run_kubernetes_tests.sh
109109

110110
# Run the Confidential Containers AMD SEV specific tests.

Diff for: ‎integration/confidential/lib.sh

+101-7
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
1212
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
1313
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
1414
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
15+
NYDUS_SNAPSHOTTER_BINARY="/opt/kata/bin/containerd-nydus-grpc"
16+
NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-host-sharing.toml"
17+
NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-guest-pulling.toml"
18+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
1519

1620
# Toggle between true and false the service_offload configuration of
1721
# the Kata agent.
@@ -180,6 +184,13 @@ disable_full_debug() {
180184
sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH"
181185
}
182186

187+
restart_containerd() {
188+
sudo systemctl restart containerd
189+
if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
190+
die "containerd seems not operational after restarted"
191+
fi
192+
}
193+
183194
# Configure containerd for confidential containers. Among other things, it ensures
184195
# the CRI handler is configured to deal with confidential container.
185196
#
@@ -198,15 +209,15 @@ configure_cc_containerd() {
198209
# installed via operator it will assume containerd is in right state
199210
# already.
200211
[ "${TESTS_CONFIGURE_CC_CONTAINERD:-yes}" == "yes" ] || return 0
212+
sudo iptables -w -P FORWARD ACCEPT
201213

202214
# Even if we are not saving the original file it is a good idea to
203215
# restart containerd because it might be in an inconsistent state here.
204216
sudo systemctl stop containerd
205217
sleep 5
206218
[ -n "$saved_containerd_conf_file" ] && \
207219
sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
208-
sudo systemctl start containerd
209-
waitForProcess 30 5 "sudo crictl info >/dev/null"
220+
restart_containerd
210221

211222
# Ensure the cc CRI handler is set.
212223
local cri_handler=$(sudo crictl info | \
@@ -223,11 +234,6 @@ configure_cc_containerd() {
223234
sudo tee -a "$containerd_conf_file"
224235
fi
225236

226-
sudo systemctl restart containerd
227-
if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
228-
die "containerd seems not operational after reconfigured"
229-
fi
230-
sudo iptables -w -P FORWARD ACCEPT
231237
}
232238

233239
#
@@ -445,3 +451,91 @@ EOF
445451
EOF
446452
fi
447453
}
454+
455+
###############################################################################
456+
457+
# remote-snapshotter
458+
459+
configure_remote_snapshotter() {
460+
case "${SNAPSHOTTER:-}" in
461+
"nydus")
462+
configure_nydus_snapshotter
463+
;;
464+
*) ;;
465+
466+
esac
467+
}
468+
469+
is_containerd_support_per_runtime_snapshotter() {
470+
containerd_version=$(containerd --version | awk '{print $3}')
471+
required_version="v1.7.0"
472+
printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet -V
473+
}
474+
475+
set_vanilla_containerd() {
476+
sudo systemctl stop containerd
477+
sleep 5
478+
sudo mv /usr/local/bin/containerd /usr/local/bin/containerd-coco
479+
sudo cp /usr/local/bin/containerd-vanilla /usr/local/bin/containerd
480+
echo "vanilla containerd version: $(containerd --version | awk '{print $3}')"
481+
}
482+
483+
unset_vanilla_containerd() {
484+
sudo systemctl stop containerd
485+
sleep 5
486+
sudo rm -f /usr/local/bin/containerd
487+
sudo mv /usr/local/bin/containerd-coco /usr/local/bin/containerd
488+
echo "coco containerd version: $(containerd --version | awk '{print $3}')"
489+
}
490+
491+
configure_containerd_for_nydus_snapshotter() {
492+
set_vanilla_containerd
493+
if [ "${SNAPSHOTTER:-}" = "nydus" ]; then
494+
local containerd_config="$1"
495+
snapshotter_socket="/run/containerd-nydus/containerd-nydus-grpc.sock"
496+
proxy_config=" [proxy_plugins.$SNAPSHOTTER]\n type = \"snapshot\"\n address = ${snapshotter_socket}"
497+
498+
if grep -q "\[proxy_plugins\]" "$containerd_config"; then
499+
sudo sed -i '/\[proxy_plugins\]/a\'"$proxy_config" "$containerd_config"
500+
else
501+
sudo echo -e "[proxy_plugins]" >>"$containerd_config"
502+
sudo echo -e "$proxy_config" >>"$containerd_config"
503+
fi
504+
505+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config"
506+
sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config"
507+
fi
508+
}
509+
510+
kill_nydus_snapshotter_process() {
511+
echo "Kill nydus snapshotter"
512+
bin="containerd-nydus-grpc"
513+
sudo kill -9 $(pidof $bin) || true
514+
sudo rm -rf "/var/lib/containerd-nydus" || true
515+
}
516+
517+
remove_test_image() {
518+
local test_image="$1"
519+
crictl rmi "$1"
520+
pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
521+
crictl rmi "$pause_name"
522+
}
523+
524+
restart_nydus_snapshotter() {
525+
kill_nydus_snapshotter_process || true
526+
echo "Restart nydus snapshotter"
527+
sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 &
528+
}
529+
530+
configure_nydus_snapshotter() {
531+
if [ "${SNAPSHOTTER:-}" = "nydus" ]; then
532+
echo "Configure nydus snapshotter"
533+
if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
534+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
535+
else
536+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
537+
sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
538+
fi
539+
restart_nydus_snapshotter
540+
fi
541+
}

Diff for: ‎integration/containerd/confidential/tests_common.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ setup_common() {
2626
echo "Prepare containerd for Confidential Container"
2727
SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
2828
configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
29-
29+
restart_containerd
3030
# Note: ensure that intructions changing the kernel parameters are
3131
# executed *after* saving the original list.
3232
saved_kernel_params=$(get_kernel_params)

Diff for: ‎integration/kubernetes/confidential/agent_image.bats

+3-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ RUNTIMECLASS="${RUNTIMECLASS:-kata}"
3131
test_tag="[cc][agent][kubernetes][containerd]"
3232

3333
setup() {
34-
setup_common
34+
setup_containerd
35+
restart_containerd
36+
reconfigure_kata
3537
}
3638

3739
@test "$test_tag Test can launch pod with measured boot enabled" {

Diff for: ‎integration/kubernetes/confidential/agent_image_encrypted.bats

+3-9
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@
44
# SPDX-License-Identifier: Apache-2.0
55
#
66

7-
load "${BATS_TEST_DIRNAME}/lib.sh"
8-
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
97
load "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
8+
load "${BATS_TEST_DIRNAME}/tests_common.sh"
109

1110
# Allow to configure the runtimeClassName on pod configuration.
1211
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
@@ -25,13 +24,8 @@ setup() {
2524
SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$"
2625
configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE"
2726

28-
echo "Reconfigure Kata Containers"
29-
switch_image_service_offload on
30-
clear_kernel_params
31-
add_kernel_params "${original_kernel_params}"
32-
33-
setup_proxy
34-
switch_measured_rootfs_verity_scheme none
27+
restart_containerd
28+
reconfigure_kata
3529
}
3630

3731
@test "$test_tag Test can pull an encrypted image inside the guest with decryption key" {

Diff for: ‎integration/kubernetes/confidential/fixtures/pod-config.yaml.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
apiVersion: v1
66
kind: Pod
77
metadata:
8-
name: busybox-cc
8+
name: busybox-cc$INDEX
99
spec:
1010
runtimeClassName: $RUNTIMECLASS
1111
containers:

Diff for: ‎integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/usr/bin/env bats
2+
# Copyright (c) 2023 Intel Corporation
3+
#
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
7+
load "${BATS_TEST_DIRNAME}/tests_common.sh"
8+
9+
tag_suffix=""
10+
if [ "$(uname -m)" != "x86_64" ]; then
11+
tag_suffix="-$(uname -m)"
12+
fi
13+
14+
# Images used on the tests.
15+
16+
image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
17+
18+
original_kernel_params=$(get_kernel_params)
19+
# Allow to configure the runtimeClassName on pod configuration.
20+
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
21+
test_tag="[cc][agent][kubernetes][containerd]"
22+
23+
setup() {
24+
remove_test_image "$image_unsigned_protected" || true
25+
setup_containerd
26+
configure_containerd_for_nydus_snapshotter "/etc/containerd/config.toml"
27+
restart_containerd
28+
reconfigure_kata
29+
switch_image_service_offload off
30+
}
31+
32+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
33+
if [ "$(uname -m)" = "s390x" ]; then
34+
skip "test for s390x as nydus-image doesn't currently support this platform"
35+
fi
36+
if [ "$SNAPSHOTTER" = "nydus" ]; then
37+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
38+
pod_config="$(new_pod_config "$image_unsigned_protected")"
39+
echo $pod_config
40+
create_test_pod
41+
fi
42+
}
43+
44+
@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" {
45+
if [ "$(uname -m)" = "s390x" ]; then
46+
skip "test for s390x as nydus-image doesn't currently support this platform"
47+
fi
48+
if [ "$SNAPSHOTTER" = "nydus" ]; then
49+
EXPORT_MODE="image_block" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
50+
pod_config="$(new_pod_config "$image_unsigned_protected")"
51+
echo $pod_config
52+
create_test_pod
53+
fi
54+
}
55+
56+
@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
57+
switch_image_service_offload on
58+
if [ "$SNAPSHOTTER" = "nydus" ]; then
59+
EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
60+
pod_config="$(new_pod_config "$image_unsigned_protected")"
61+
echo $pod_config
62+
create_test_pod
63+
fi
64+
}
65+
66+
teardown() {
67+
teardown_common
68+
remove_test_image "$image_unsigned_protected" || true
69+
kill_nydus_snapshotter_process
70+
}

Diff for: ‎integration/kubernetes/confidential/lib.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ kubernetes_create_cc_pod() {
4646
fi
4747

4848
kubectl apply -f ${config_file}
49-
if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
49+
pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name')
50+
if ! kubectl get pod "$pod_name" &> /dev/null; then
5051
echo "Failed to create the pod"
5152
return 1
5253
fi

0 commit comments

Comments
 (0)