cli: Update CI test framework to sync with driver v0.5 integration

The CI framework must be updated taking into consideration that it is
no longer possible to run the power-user version of the CLI while the
Nitro Enclaves driver is inserted (since the latter takes exclusive
ownership of the communication with the PCI device).

Signed-off-by: Andrei Trandafir <[email protected]>

Author: Andrei Trandafir

Makefile

@@ -106,12 +106,12 @@ nitro_cli_resource_allocator: drivers/nitro_cli_resource_allocator/nitro_cli_res
 nitro_cli_resource_allocator-clean:
 	PREV_DIR=$$PWD && cd drivers/nitro_cli_resource_allocator/ && make clean && cd $$PREV_DIR \
 
-nitro_enclaves: drivers/virt/amazon/nitro_enclaves/ne_main.c driver-deps
-	PREV_DIR=$$PWD && cd drivers/virt/amazon/nitro_enclaves/ && make && cd $$PREV_DIR
+nitro_enclaves: drivers/virt/nitro_enclaves/ne_misc_dev.c drivers/virt/nitro_enclaves/ne_pci_dev.c driver-deps
+	PREV_DIR=$$PWD && cd drivers/virt/nitro_enclaves/ && make && cd $$PREV_DIR
 
 .PHONY: nitro_enclaves-clean
 nitro_enclaves-clean:
-	PREV_DIR=$$PWD && cd drivers/virt/amazon/nitro_enclaves/ && make clean && cd $$PREV_DIR
+	PREV_DIR=$$PWD && cd drivers/virt/nitro_enclaves/ && make clean && cd $$PREV_DIR
 
 .PHONY: driver-clean
 driver-clean: nitro_enclaves-clean nitro_cli_resource_allocator-clean
@@ -183,8 +183,9 @@ command-executer: build-setup build-container .build-command-executer
 				--target-dir=/nitro_build/nitro_cli \
 				--message-format json \
 				| tee /nitro_build/nitro-tests-build.log | \
-				jq -r "select(.profile.test == true) | .filenames[]" \
-					 > /nitro_build/test_executables.txt && \
+				jq -r "select(.profile.test == true) | .filenames[], .package_id" | \
+				paste -d " " - - | cut -d " " -f 1,2 \
+					> /nitro_build/test_executables.txt && \
 			chmod -R 777 nitro_build '
 	touch $@
 
@@ -277,9 +278,10 @@ install: install-tools nitro_enclaves
 	$(INSTALL) -D -m 0755 blobs/nsm.ko ${NITRO_CLI_INSTALL_DIR}/${OPT_DIR}/nitro_cli/nsm.ko
 	$(INSTALL) -D -m 0755 blobs/linuxkit ${NITRO_CLI_INSTALL_DIR}/${OPT_DIR}/nitro_cli/linuxkit
 	$(MKDIR) -p ${NITRO_CLI_INSTALL_DIR}/lib/modules/$(uname -r)/extra/nitro_enclaves
-	$(INSTALL) -D -m 0755 drivers/virt/amazon/nitro_enclaves/nitro_enclaves.ko \
+	$(INSTALL) -D -m 0755 drivers/virt/nitro_enclaves/nitro_enclaves.ko \
                ${NITRO_CLI_INSTALL_DIR}/lib/modules/$(uname -r)/extra/nitro_enclaves/nitro_enclaves.ko
 	$(INSTALL) -m 0644 tools/env.sh ${NITRO_CLI_INSTALL_DIR}/${ENV_SETUP_DIR}/nitro-cli-env.sh
+	$(INSTALL) -m 0744 tools/nitro-cli-config.sh ${NITRO_CLI_INSTALL_DIR}/${ENV_SETUP_DIR}/nitro-cli-config.sh
 	sed -i "2 a NITRO_CLI_INSTALL_DIR=$$(readlink -f ${NITRO_CLI_INSTALL_DIR})" \
 		${NITRO_CLI_INSTALL_DIR}/${ENV_SETUP_DIR}/nitro-cli-env.sh
 	echo "Installation finished"
@@ -295,6 +297,7 @@ uninstall:
 	$(RM) -rf ${NITRO_CLI_INSTALL_DIR}/lib/modules/$(uname -r)/extra/nitro_enclaves
 	$(RM) -f ${NITRO_CLI_INSTALL_DIR}/${CONF_DIR}/vsock_proxy/config.yaml
 	$(RM) -f ${NITRO_CLI_INSTALL_DIR}/${ENV_SETUP_DIR}/nitro-cli-env.sh
+	$(RM) -f ${NITRO_CLI_INSTALL_DIR}/${ENV_SETUP_DIR}/nitro-cli-config.sh
 
 .PHONY: clean
 clean:

run_tests.sh

@@ -22,8 +22,8 @@ function register_test_fail() {
 
 # Clean up and exit with the current test suite's status
 function clean_up_and_exit() {
-	rmmod nitro_cli_resource_allocator || register_test_fail
-	rmmod nitro_enclaves || register_test_fail
+	[ "$(lsmod | grep -cw nitro_cli_resource_allocator)" -eq 0 ] || rmmod nitro_cli_resource_allocator || register_test_fail
+	[ "$(lsmod | grep -cw nitro_enclaves)" -eq 0 ] || rmmod nitro_enclaves || register_test_fail
 	make clean
 	rm -rf test_images
 
@@ -36,9 +36,23 @@ function test_failed() {
 	clean_up_and_exit
 }
 
+# Remove the Nitro Enclaves driver
+function remove_ne_driver() {
+	[ "$(lsmod | grep -cw nitro_enclaves)" -eq 0 ] || rmmod nitro_enclaves || test_failed
+}
+
+# Configure and insert the Nitro Enclaves driver
+function configure_ne_driver() {
+	if [ "$(lsmod | grep -cw nitro_enclaves)" -eq 0 ]
+	then
+		# Preallocate 2048 Mb, that should be enough for all the tests
+		source build/install/etc/profile.d/nitro-cli-env.sh || test_failed
+		./build/install/etc/profile.d/nitro-cli-config.sh -m 2048 -p 1,3 || test_failed
+	fi
+}
+
 # First run the instalation test, before we change the environement
-pytest-3 tests/integration/test_installation.py \
-	|| test_failed
+pytest-3 tests/integration/test_installation.py || test_failed
 
 # Clean up build artefacts
 make clean
@@ -64,30 +78,40 @@ make vsock-proxy || test_failed
 make install || test_failed
 insmod drivers/nitro_cli_resource_allocator/nitro_cli_resource_allocator.ko || test_failed
 
-# Preallocate 2048 Mb, that should be enough for all the tests
-echo 1024 > /proc/sys/vm/nr_hugepages || test_failed
-source build/install/etc/profile.d/nitro-cli-env.sh || test_failed
+# Ensure the Nitro Enclaves driver is inserted at the beginning.
+configure_ne_driver
 
 # Create directories for enclave process sockets and logs
 mkdir -p /var/run/nitro_enclaves || test_failed
 mkdir -p /var/log/nitro_enclaves || test_failed
 
 # Build EIFS for testing
 mkdir -p test_images
-nitro-cli build-enclave --docker-uri 667861386598.dkr.ecr.us-east-1.amazonaws.com/enclaves-samples:vsock-sample --output-file test_images/vsock-sample.eif
+nitro-cli build-enclave --docker-uri 667861386598.dkr.ecr.us-east-1.amazonaws.com/enclaves-samples:vsock-sample \
+	--output-file test_images/vsock-sample.eif || test_failed
 
 # Run all unit tests
-while IFS= read -r test_exec_path
+while IFS= read -r test_line
 do
 	TEST_SUITES_TOTAL=$((TEST_SUITES_TOTAL + 1))
-	test_exec_name=$(basename "${test_exec_path}")
-	RUST_BACKTRACE=1 ./build/nitro_cli/x86_64-unknown-linux-musl/release/"${test_exec_name}" \
-		--test-threads=1 --nocapture \
-		|| test_failed
+	test_module="$(echo ${test_line} | cut -d' ' -f2)"
+	test_exec_name="$(basename $(echo ${test_line} | cut -d' ' -f1))"
+
+	if [[ $test_module == *"nitro-cli-poweruser"* ]]
+	then
+		remove_ne_driver
+	else
+		configure_ne_driver
+	fi
+
+	./build/nitro_cli/x86_64-unknown-linux-musl/release/"${test_exec_name}" \
+		--test-threads=1 --nocapture || test_failed
 done < <(grep -v '^ *#' < build/test_executables.txt)
 
+# Ensure the Nitro Enclaves driver is inserted for the remaining integration tests.
+configure_ne_driver
+
 # Run integration tests except the instalation test
-pytest-3 tests/integration/ --ignore tests/integration/test_installation.py \
-	|| test_failed
+pytest-3 tests/integration/ --ignore tests/integration/test_installation.py || test_failed
 
 clean_up_and_exit

tests/integration/helpers.py

@@ -134,3 +134,11 @@ def connect_console(enclave_id):
                 "--enclave-id", enclave_id]
 
         return subprocess.Popen(args, stdout = PIPE, stderr = PIPE)
+
+# Get the number of CPUs in the system, both on-line and off-line.
+def get_cpu_count():
+        run_lscpu = subprocess.Popen(["lscpu -a -p=cpu"], stdout=subprocess.PIPE, shell=True)
+        (output, _) = run_lscpu.communicate()
+        output = output.decode('UTF-8').splitlines()
+        cpu_ids = [id for id in output if not id.startswith("#")]
+        return len(cpu_ids)

tests/integration/test_commands.py

@@ -64,7 +64,7 @@ def test_run_invalid_cpu_count(init_resources):
         result = run_enclave_err(SAMPLE_EIF, "1028", "3")
         result = run_enclave_err(SAMPLE_EIF, "1028", "-3")
         result = run_enclave_err(SAMPLE_EIF, "1028", "zzz")
-        result = run_enclave_err(SAMPLE_EIF, "1028", str(os.cpu_count()))
+        result = run_enclave_err(SAMPLE_EIF, "1028", str(get_cpu_count()))
 
         # At the end check we can still launch enclaves.
         result = run_enclave_ok(SAMPLE_EIF, "1028", "2")

tests/integration/test_installation.py

@@ -35,6 +35,11 @@ def test_install_uninstall():
                        "/etc/profile.d//nitro-cli-env.sh && nitro-cli --help && vsock-proxy --help\""
         helpers.run_cmd_ok(help_cmd)
 
+        # Perform clean-up.
         uninstall_cmd = make + " uninstall"
         helpers.run_cmd_ok(uninstall_cmd)
         helpers.check_no_files(install_dir.name)
+
+        # Also remove the inserted driver.
+        rmmod_cmd = "rmmod nitro_enclaves"
+        helpers.run_cmd_ok(rmmod_cmd)

tools/env.sh

@@ -1,4 +1,7 @@
+#!/bin/bash
 
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
 
 if [ -z ${NITRO_CLI_INSTALL_DIR} ];
 then

tools/nitro-cli-config.sh

@@ -126,7 +126,8 @@ function print_usage {
     echo -e "\t-r: Remove the driver."
     echo -e "\t-h: Print these help messages."
     echo -e "\t-m: The amount of memory that will be needed for running enclaves, in megabytes."
-    echo -e "\t-p: The CPU pool that taken from the parent instance and made available for enclaves."
+    echo -e "\t-p: The CPU pool that is taken from the parent instance and made available for enclaves. The pool format"
+    echo -e "\t    is given in: https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html#cpu-lists"
 }
 
 # Verify that the provided driver directory is correct.