Skip to content

Commit cbe3231

Browse files
author
ChengyuZhu6
committed
ci: Add test cases for CoCo image pulling without forked containerd
Add test cases for CoCo image pulling without forked containerd, including: 1) sharing image on the host with dm-verity 2) pulling image in the guest Fixes: kata-containers#5763 Depends-on: github.com/kata-containers/kata-containers#7676 github.com/kata-containers/kata-containers#7688 Signed-off-by: ChengyuZhu6 <[email protected]>
1 parent 6ab2a91 commit cbe3231

File tree

2 files changed

+163
-101
lines changed

2 files changed

+163
-101
lines changed

Diff for: ‎integration/confidential/lib.sh

+142-97
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
1212
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
1313
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
1414
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
15+
NYDUS_SNAPSHOTTER_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config.toml"
16+
CONTAINERD_CONFIG="/etc/containerd/config.toml"
1517

1618
# Toggle between true and false the service_offload configuration of
1719
# the Kata agent.
@@ -29,18 +31,18 @@ switch_image_service_offload() {
2931
load_runtime_config_path
3032

3133
case "$1" in
32-
"on")
33-
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
34-
"$RUNTIME_CONFIG_PATH"
35-
;;
36-
"off")
37-
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
38-
"$RUNTIME_CONFIG_PATH"
39-
40-
;;
41-
*)
42-
die "Unknown option '$1'"
43-
;;
34+
"on")
35+
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
36+
"$RUNTIME_CONFIG_PATH"
37+
;;
38+
"off")
39+
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
40+
"$RUNTIME_CONFIG_PATH"
41+
42+
;;
43+
*)
44+
die "Unknown option '$1'"
45+
;;
4446
esac
4547
}
4648

@@ -66,13 +68,13 @@ switch_measured_rootfs_verity_scheme() {
6668
load_runtime_config_path
6769

6870
case "$1" in
69-
"dm-verity"|"none")
70-
sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
71-
"$RUNTIME_CONFIG_PATH"
72-
;;
73-
*)
74-
die "Unknown option '$1'"
75-
;;
71+
"dm-verity" | "none")
72+
sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
73+
"$RUNTIME_CONFIG_PATH"
74+
;;
75+
*)
76+
die "Unknown option '$1'"
77+
;;
7678
esac
7779
}
7880

@@ -110,8 +112,8 @@ add_kernel_params() {
110112
get_kernel_params() {
111113
load_runtime_config_path
112114

113-
local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
114-
"$RUNTIME_CONFIG_PATH")
115+
local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
116+
"$RUNTIME_CONFIG_PATH")
115117
echo "$kernel_params"
116118
}
117119

@@ -203,13 +205,13 @@ configure_cc_containerd() {
203205
# restart containerd because it might be in an inconsistent state here.
204206
sudo systemctl stop containerd
205207
sleep 5
206-
[ -n "$saved_containerd_conf_file" ] && \
208+
[ -n "$saved_containerd_conf_file" ] &&
207209
sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
208210
sudo systemctl start containerd
209211
waitForProcess 30 5 "sudo crictl info >/dev/null"
210212

211213
# Ensure the cc CRI handler is set.
212-
local cri_handler=$(sudo crictl info | \
214+
local cri_handler=$(sudo crictl info |
213215
jq '.config.containerd.runtimes.kata.cri_handler')
214216
if [[ ! "$cri_handler" =~ cc ]]; then
215217
sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
@@ -219,8 +221,8 @@ configure_cc_containerd() {
219221
if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
220222
echo " [plugins.cri.cni]
221223
# conf_dir is the directory in which the admin places a CNI conf.
222-
conf_dir = \"/etc/cni/net.d\"" | \
223-
sudo tee -a "$containerd_conf_file"
224+
conf_dir = \"/etc/cni/net.d\"" |
225+
sudo tee -a "$containerd_conf_file"
224226
fi
225227

226228
sudo systemctl restart containerd
@@ -286,38 +288,36 @@ setup_cosign_signatures_files() {
286288

287289
# Set-up required files in guest image
288290
case "${AA_KBC:-}" in
289-
"offline_fs_kbc")
290-
add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
291-
cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
292-
;;
293-
"cc_kbc")
294-
# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
295-
# by the service, as well as the one configured in the Kata Containers rootfs.
296-
297-
CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
298-
CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
299-
add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
300-
;;
301-
*)
302-
;;
291+
"offline_fs_kbc")
292+
add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
293+
cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
294+
;;
295+
"cc_kbc")
296+
# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
297+
# by the service, as well as the one configured in the Kata Containers rootfs.
298+
299+
CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
300+
CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
301+
add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
302+
;;
303+
*) ;;
303304
esac
304305
}
305306

306307
setup_signature_files() {
307308
case "${AA_KBC:-}" in
308-
"offline_fs_kbc")
309-
setup_offline_fs_kbc_signature_files_in_guest
310-
;;
311-
"cc_kbc")
312-
setup_cc_kbc_signature_files_in_guest
313-
;;
314-
*)
315-
;;
309+
"offline_fs_kbc")
310+
setup_offline_fs_kbc_signature_files_in_guest
311+
;;
312+
"cc_kbc")
313+
setup_cc_kbc_signature_files_in_guest
314+
;;
315+
*) ;;
316316
esac
317317
}
318318

319319
# In case the tests run behind a firewall where images needed to be fetched
320-
# through a proxy.
320+
# through a proxy.
321321
# Note: With measured rootfs enabled, we can not set proxy through
322322
# agent config file.
323323
setup_proxy() {
@@ -349,8 +349,8 @@ setup_credentials_files() {
349349

350350
dest_dir="$(mktemp -t -d offline-fs-kbc-XXXXXXXX)"
351351
dest_file=${dest_dir}/aa-offline_fs_kbc-resources.json
352-
auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
353-
CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}"
352+
auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
353+
CREDENTIAL="${auth_json}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" >"${dest_file}"
354354
cp_to_guest_img "etc" "${dest_file}"
355355
}
356356

@@ -366,77 +366,122 @@ KBS_DB="${KBS_DB:-simple_kbs}"
366366

367367
# Run the simple-kbs
368368
simple_kbs_run() {
369-
# Retrieve simple-kbs repo and tag from versions.yaml
370-
local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
371-
local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
372-
373-
# Cleanup and create installation directory
374-
esudo rm -rf "${SIMPLE_KBS_DIR}"
375-
mkdir -p "${SIMPLE_KBS_DIR}"
376-
pushd "${SIMPLE_KBS_DIR}"
377-
378-
# Clone and run
379-
git clone "${simple_kbs_url}" --branch main
380-
pushd simple-kbs
381-
382-
# Checkout, build and start
383-
git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
384-
esudo docker-compose build
385-
esudo docker-compose up -d
386-
387-
# Wait for simple-kbs to start
388-
waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
389-
popd
390-
391-
# Get simple-kbs database container ip
392-
local kbs_db_host=$(simple_kbs_get_db_ip)
393-
394-
# Confirm connection to the database is possible
395-
waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
396-
popd
369+
# Retrieve simple-kbs repo and tag from versions.yaml
370+
local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
371+
local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
372+
373+
# Cleanup and create installation directory
374+
esudo rm -rf "${SIMPLE_KBS_DIR}"
375+
mkdir -p "${SIMPLE_KBS_DIR}"
376+
pushd "${SIMPLE_KBS_DIR}"
377+
378+
# Clone and run
379+
git clone "${simple_kbs_url}" --branch main
380+
pushd simple-kbs
381+
382+
# Checkout, build and start
383+
git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
384+
esudo docker-compose build
385+
esudo docker-compose up -d
386+
387+
# Wait for simple-kbs to start
388+
waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
389+
popd
390+
391+
# Get simple-kbs database container ip
392+
local kbs_db_host=$(simple_kbs_get_db_ip)
393+
394+
# Confirm connection to the database is possible
395+
waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
396+
popd
397397
}
398398

399399
# Stop simple-kbs and database containers
400400
simple_kbs_stop() {
401-
(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
401+
(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
402402
}
403403

404404
# Delete all test inserted data in the simple-kbs
405405
simple_kbs_delete_data() {
406-
# Get simple-kbs database container ip
407-
local kbs_db_host=$(simple_kbs_get_db_ip)
406+
# Get simple-kbs database container ip
407+
local kbs_db_host=$(simple_kbs_get_db_ip)
408408

409-
# Delete all data with 'id = 10'
410-
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
409+
# Delete all data with 'id = 10'
410+
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
411411
DELETE FROM secrets WHERE id = 10;
412412
DELETE FROM policy WHERE id = 10;
413413
EOF
414414
}
415415

416416
# Get the ip of the simple-kbs database docker container
417417
simple_kbs_get_db_ip() {
418-
esudo docker network inspect simple-kbs_default \
419-
| jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' \
420-
| sed "s|/.*$||g"
418+
esudo docker network inspect simple-kbs_default |
419+
jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' |
420+
sed "s|/.*$||g"
421421
}
422422

423423
# Add key and keyset to database
424424
# If measurement is provided, add policy with measurement to database
425425
simple_kbs_add_key_to_db() {
426-
local encryption_key="${1}"
427-
local measurement="${2}"
428-
429-
# Get simple-kbs database container ip
430-
local kbs_db_host=$(simple_kbs_get_db_ip)
431-
432-
if [ -n "${measurement}" ]; then
433-
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
426+
local encryption_key="${1}"
427+
local measurement="${2}"
428+
429+
# Get simple-kbs database container ip
430+
local kbs_db_host=$(simple_kbs_get_db_ip)
431+
432+
if [ -n "${measurement}" ]; then
433+
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
434434
INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', 10);
435435
INSERT INTO policy VALUES (10, '["${measurement}"]', '[]', 0, 0, '[]', now(), NULL, 1);
436436
EOF
437-
else
438-
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
437+
else
438+
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
439439
INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', NULL);
440440
EOF
441-
fi
441+
fi
442+
}
443+
444+
###############################################################################
445+
446+
# remote-snapshotter
447+
448+
configure_remote_snapshotter() {
449+
case "${SNAPSHOTTER:-}" in
450+
"nydus")
451+
configure_nydus_snapshotter
452+
;;
453+
*) ;;
454+
455+
esac
456+
}
457+
check_containerd_version() {
458+
containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
459+
if echo $containerd_version | grep -q "^v1.7"; then
460+
return 1
461+
else
462+
return 0
463+
fi
464+
}
465+
configure_containerd_for_nydus_snapshotter() {
466+
if check_containerd_version; then
467+
sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n disable_snapshot_annotations = false' "$CONTAINERD_CONFIG"
468+
else
469+
sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g; s/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
470+
fi
471+
}
472+
remove_nydus_snapshotter_from_containerd() {
473+
if check_containerd_version; then
474+
sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;/disable_snapshot_annotations = false/d;}' "$CONTAINERD_CONFIG"
475+
else
476+
sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g; s/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
477+
fi
478+
}
479+
restart_nydus_snapshotter() {
480+
echo "Restart nydus snapshotter"
481+
}
482+
483+
configure_nydus_snapshotter() {
484+
echo "Configure nydus snapshotter"
485+
sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" $NYDUS_SNAPSHOTTER_CONFIG
486+
restart_nydus_snapshotter
442487
}

Diff for: ‎integration/kubernetes/confidential/agent_image.bats

+21-4
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ if [ "$(uname -m)" != "x86_64" ]; then
1313
fi
1414

1515
# Images used on the tests.
16-
## Cosign
16+
## Cosign
1717
image_cosigned="quay.io/kata-containers/confidential-containers:cosign-signed${tag_suffix}"
1818
image_cosigned_other="quay.io/kata-containers/confidential-containers:cosign-signed-key2"
1919

@@ -70,7 +70,7 @@ new_pod_config() {
7070
local image="$1"
7171

7272
local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
73-
IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config"
73+
IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst <"$base_config" >"$new_config"
7474
echo "$new_config"
7575
}
7676

@@ -90,7 +90,7 @@ setup() {
9090
switch_image_service_offload on
9191
clear_kernel_params
9292
add_kernel_params "${original_kernel_params}"
93-
93+
9494
setup_proxy
9595
switch_measured_rootfs_verity_scheme none
9696
}
@@ -183,7 +183,6 @@ assert_logs_contain() {
183183
assert_logs_contain 'Validate image failed: \[PublicKeyVerifier { key: ECDSA_P256_SHA256_ASN1'
184184
}
185185

186-
187186
@test "$test_tag Test pull an unencrypted unsigned image from an authenticated registry with correct credentials" {
188187
if [ "${AA_KBC}" = "offline_fs_kbc" ]; then
189188
setup_credentials_files "quay.io/kata-containers/confidential-containers-auth"
@@ -223,6 +222,24 @@ assert_logs_contain() {
223222
assert_logs_contain 'failed to pull manifest Not authorized'
224223
}
225224

225+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
226+
if [ "${SNAPSHOTTER}" = "nydus" ]; then
227+
EXPORT_MODE="image_block_with_verity" configure_remote_snapshotter
228+
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
229+
echo $pod_config
230+
create_test_pod
231+
fi
232+
}
233+
234+
@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
235+
if [ "${SNAPSHOTTER}" = "nydus" ]; then
236+
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter
237+
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
238+
echo $pod_config
239+
create_test_pod
240+
fi
241+
}
242+
226243
teardown() {
227244
# Print the logs and cleanup resources.
228245
echo "-- Kata logs:"

0 commit comments

Comments
 (0)