Skip to content

Commit 3c8c4f7

Browse files
author
ChengyuZhu6
committed
ci: Add test cases for CoCo image pulling without forked containerd
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. Fixes kata-containers#5763 Depends: kata-containers/kata-containers#7688 kata-containers/kata-containers#7676 Signed-off-by: ChengyuZhu6 <[email protected]>
1 parent 021bf58 commit 3c8c4f7

File tree

3 files changed

+319
-98
lines changed

3 files changed

+319
-98
lines changed

Diff for: ‎integration/confidential/lib.sh

+163-97
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
1212
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
1313
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
1414
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
15+
NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config_coco_tarfs.toml"
16+
NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config-coco-guest-pulling.toml"
17+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
18+
NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc"
19+
CONTAINERD_CONFIG="/etc/containerd/config.toml"
1520

1621
# Toggle between true and false the service_offload configuration of
1722
# the Kata agent.
@@ -29,18 +34,18 @@ switch_image_service_offload() {
2934
load_runtime_config_path
3035

3136
case "$1" in
32-
"on")
33-
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
34-
"$RUNTIME_CONFIG_PATH"
35-
;;
36-
"off")
37-
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
38-
"$RUNTIME_CONFIG_PATH"
39-
40-
;;
41-
*)
42-
die "Unknown option '$1'"
43-
;;
37+
"on")
38+
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
39+
"$RUNTIME_CONFIG_PATH"
40+
;;
41+
"off")
42+
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
43+
"$RUNTIME_CONFIG_PATH"
44+
45+
;;
46+
*)
47+
die "Unknown option '$1'"
48+
;;
4449
esac
4550
}
4651

@@ -66,13 +71,13 @@ switch_measured_rootfs_verity_scheme() {
6671
load_runtime_config_path
6772

6873
case "$1" in
69-
"dm-verity"|"none")
70-
sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
71-
"$RUNTIME_CONFIG_PATH"
72-
;;
73-
*)
74-
die "Unknown option '$1'"
75-
;;
74+
"dm-verity" | "none")
75+
sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
76+
"$RUNTIME_CONFIG_PATH"
77+
;;
78+
*)
79+
die "Unknown option '$1'"
80+
;;
7681
esac
7782
}
7883

@@ -110,8 +115,8 @@ add_kernel_params() {
110115
get_kernel_params() {
111116
load_runtime_config_path
112117

113-
local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
114-
"$RUNTIME_CONFIG_PATH")
118+
local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
119+
"$RUNTIME_CONFIG_PATH")
115120
echo "$kernel_params"
116121
}
117122

@@ -203,13 +208,13 @@ configure_cc_containerd() {
203208
# restart containerd because it might be in an inconsistent state here.
204209
sudo systemctl stop containerd
205210
sleep 5
206-
[ -n "$saved_containerd_conf_file" ] && \
211+
[ -n "$saved_containerd_conf_file" ] &&
207212
sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
208213
sudo systemctl start containerd
209214
waitForProcess 30 5 "sudo crictl info >/dev/null"
210215

211216
# Ensure the cc CRI handler is set.
212-
local cri_handler=$(sudo crictl info | \
217+
local cri_handler=$(sudo crictl info |
213218
jq '.config.containerd.runtimes.kata.cri_handler')
214219
if [[ ! "$cri_handler" =~ cc ]]; then
215220
sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
@@ -219,8 +224,8 @@ configure_cc_containerd() {
219224
if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
220225
echo " [plugins.cri.cni]
221226
# conf_dir is the directory in which the admin places a CNI conf.
222-
conf_dir = \"/etc/cni/net.d\"" | \
223-
sudo tee -a "$containerd_conf_file"
227+
conf_dir = \"/etc/cni/net.d\"" |
228+
sudo tee -a "$containerd_conf_file"
224229
fi
225230

226231
sudo systemctl restart containerd
@@ -286,38 +291,36 @@ setup_cosign_signatures_files() {
286291

287292
# Set-up required files in guest image
288293
case "${AA_KBC:-}" in
289-
"offline_fs_kbc")
290-
add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
291-
cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
292-
;;
293-
"cc_kbc")
294-
# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
295-
# by the service, as well as the one configured in the Kata Containers rootfs.
296-
297-
CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
298-
CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
299-
add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
300-
;;
301-
*)
302-
;;
294+
"offline_fs_kbc")
295+
add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
296+
cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
297+
;;
298+
"cc_kbc")
299+
# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
300+
# by the service, as well as the one configured in the Kata Containers rootfs.
301+
302+
CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
303+
CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
304+
add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
305+
;;
306+
*) ;;
303307
esac
304308
}
305309

306310
setup_signature_files() {
307311
case "${AA_KBC:-}" in
308-
"offline_fs_kbc")
309-
setup_offline_fs_kbc_signature_files_in_guest
310-
;;
311-
"cc_kbc")
312-
setup_cc_kbc_signature_files_in_guest
313-
;;
314-
*)
315-
;;
312+
"offline_fs_kbc")
313+
setup_offline_fs_kbc_signature_files_in_guest
314+
;;
315+
"cc_kbc")
316+
setup_cc_kbc_signature_files_in_guest
317+
;;
318+
*) ;;
316319
esac
317320
}
318321

319322
# In case the tests run behind a firewall where images needed to be fetched
320-
# through a proxy.
323+
# through a proxy.
321324
# Note: With measured rootfs enabled, we can not set proxy through
322325
# agent config file.
323326
setup_proxy() {
@@ -349,8 +352,8 @@ setup_credentials_files() {
349352

350353
dest_dir="$(mktemp -t -d offline-fs-kbc-XXXXXXXX)"
351354
dest_file=${dest_dir}/aa-offline_fs_kbc-resources.json
352-
auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
353-
CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}"
355+
auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
356+
CREDENTIAL="${auth_json}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" >"${dest_file}"
354357
cp_to_guest_img "etc" "${dest_file}"
355358
}
356359

@@ -366,77 +369,140 @@ KBS_DB="${KBS_DB:-simple_kbs}"
366369

367370
# Run the simple-kbs
368371
simple_kbs_run() {
369-
# Retrieve simple-kbs repo and tag from versions.yaml
370-
local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
371-
local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
372-
373-
# Cleanup and create installation directory
374-
esudo rm -rf "${SIMPLE_KBS_DIR}"
375-
mkdir -p "${SIMPLE_KBS_DIR}"
376-
pushd "${SIMPLE_KBS_DIR}"
377-
378-
# Clone and run
379-
git clone "${simple_kbs_url}" --branch main
380-
pushd simple-kbs
381-
382-
# Checkout, build and start
383-
git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
384-
esudo docker-compose build
385-
esudo docker-compose up -d
386-
387-
# Wait for simple-kbs to start
388-
waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
389-
popd
390-
391-
# Get simple-kbs database container ip
392-
local kbs_db_host=$(simple_kbs_get_db_ip)
393-
394-
# Confirm connection to the database is possible
395-
waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
396-
popd
372+
# Retrieve simple-kbs repo and tag from versions.yaml
373+
local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
374+
local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
375+
376+
# Cleanup and create installation directory
377+
esudo rm -rf "${SIMPLE_KBS_DIR}"
378+
mkdir -p "${SIMPLE_KBS_DIR}"
379+
pushd "${SIMPLE_KBS_DIR}"
380+
381+
# Clone and run
382+
git clone "${simple_kbs_url}" --branch main
383+
pushd simple-kbs
384+
385+
# Checkout, build and start
386+
git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
387+
esudo docker-compose build
388+
esudo docker-compose up -d
389+
390+
# Wait for simple-kbs to start
391+
waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
392+
popd
393+
394+
# Get simple-kbs database container ip
395+
local kbs_db_host=$(simple_kbs_get_db_ip)
396+
397+
# Confirm connection to the database is possible
398+
waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
399+
popd
397400
}
398401

399402
# Stop simple-kbs and database containers
400403
simple_kbs_stop() {
401-
(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
404+
(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
402405
}
403406

404407
# Delete all test inserted data in the simple-kbs
405408
simple_kbs_delete_data() {
406-
# Get simple-kbs database container ip
407-
local kbs_db_host=$(simple_kbs_get_db_ip)
409+
# Get simple-kbs database container ip
410+
local kbs_db_host=$(simple_kbs_get_db_ip)
408411

409-
# Delete all data with 'id = 10'
410-
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
412+
# Delete all data with 'id = 10'
413+
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
411414
DELETE FROM secrets WHERE id = 10;
412415
DELETE FROM policy WHERE id = 10;
413416
EOF
414417
}
415418

416419
# Get the ip of the simple-kbs database docker container
417420
simple_kbs_get_db_ip() {
418-
esudo docker network inspect simple-kbs_default \
419-
| jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' \
420-
| sed "s|/.*$||g"
421+
esudo docker network inspect simple-kbs_default |
422+
jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' |
423+
sed "s|/.*$||g"
421424
}
422425

423426
# Add key and keyset to database
424427
# If measurement is provided, add policy with measurement to database
425428
simple_kbs_add_key_to_db() {
426-
local encryption_key="${1}"
427-
local measurement="${2}"
428-
429-
# Get simple-kbs database container ip
430-
local kbs_db_host=$(simple_kbs_get_db_ip)
431-
432-
if [ -n "${measurement}" ]; then
433-
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
429+
local encryption_key="${1}"
430+
local measurement="${2}"
431+
432+
# Get simple-kbs database container ip
433+
local kbs_db_host=$(simple_kbs_get_db_ip)
434+
435+
if [ -n "${measurement}" ]; then
436+
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
434437
INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', 10);
435438
INSERT INTO policy VALUES (10, '["${measurement}"]', '[]', 0, 0, '[]', now(), NULL, 1);
436439
EOF
437-
else
438-
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
440+
else
441+
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
439442
INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', NULL);
440443
EOF
441-
fi
444+
fi
445+
}
446+
447+
###############################################################################
448+
449+
# remote-snapshotter
450+
451+
configure_remote_snapshotter() {
452+
case "${SNAPSHOTTER:-}" in
453+
"nydus")
454+
configure_nydus_snapshotter
455+
;;
456+
*) ;;
457+
458+
esac
459+
}
460+
check_containerd_version() {
461+
containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
462+
if echo $containerd_version | grep -q "^v1.7"; then
463+
return 1
464+
else
465+
return 0
466+
fi
467+
}
468+
configure_containerd_for_nydus_snapshotter() {
469+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG"
470+
if check_containerd_version; then
471+
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$CONTAINERD_CONFIG"
472+
else
473+
sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
474+
fi
475+
}
476+
remove_nydus_snapshotter_from_containerd() {
477+
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG"
478+
if check_containerd_version; then
479+
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG"
480+
else
481+
sudo sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
482+
fi
483+
}
484+
remove_test_image() {
485+
local test_image = "$1"
486+
sudo crictl rmi "$1"
487+
pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
488+
sudo crictl rmi "$pause_name"
489+
}
490+
491+
restart_nydus_snapshotter() {
492+
echo "Kill nydus snapshotter"
493+
bin="containerd-nydus-grpc"
494+
sudo kill -9 $(pidof $bin) || true
495+
echo "Restart nydus snapshotter"
496+
sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" &
497+
}
498+
499+
configure_nydus_snapshotter() {
500+
echo "Configure nydus snapshotter"
501+
if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
502+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
503+
else
504+
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
505+
fi
506+
sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
507+
restart_nydus_snapshotter
442508
}

Diff for: ‎integration/kubernetes/confidential/fixtures/pod-config.yaml.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
apiVersion: v1
66
kind: Pod
77
metadata:
8-
name: busybox-cc
8+
name: busybox-cc$INDEX
99
spec:
1010
runtimeClassName: $RUNTIMECLASS
1111
containers:

0 commit comments

Comments
 (0)