ci(performance): Add performance tests to CI

Author: lucasssvaz

.github/scripts/tests_build.sh

@@ -2,19 +2,20 @@
 
 USAGE="
 USAGE:
-    ${0} -c <chunk_build_opts>
-       Example: ${0} -c -t esp32 -i 0 -m 15
+    ${0} -c -type <test_type> <chunk_build_opts>
+       Example: ${0} -c -type validation -t esp32 -i 0 -m 15
     ${0} -s sketch_name <build_opts>
        Example: ${0} -s hello_world -t esp32
     ${0} -clean
        Remove build and test generated files
 "
 
 function clean(){
-    rm -rf tests/*/build*/
     rm -rf tests/.pytest_cache
-    rm -rf tests/*/__pycache__/
-    rm -rf tests/*/*.xml
+    find tests/ -type d -name 'build*' -exec rm -rf "{}" \+
+    find tests/ -type d -name '__pycache__' -exec rm -rf "{}" \+
+    find tests/ -name '*.xml' -exec rm -rf "{}" \+
+    find tests/ -name 'result_*.json' -exec rm -rf "{}" \+
 }
 
 SCRIPTS_DIR="./.github/scripts"
@@ -35,6 +36,10 @@ while [ ! -z "$1" ]; do
         echo "$USAGE"
         exit 0
         ;;
+    -type )
+        shift
+        test_type=$1
+        ;;
     -clean )
         clean
         exit 0
@@ -52,12 +57,25 @@ source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
 args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH"
 
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+    if [ -n "$sketch" ]; then
+        tmp_sketch_path=$(find tests -name $sketch.ino)
+        test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+        echo "Sketch $sketch test type: $test_type"
+        test_folder="$PWD/tests/$test_type"
+    else
+      test_folder="$PWD/tests"
+    fi
+else
+    test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_build -eq 1 ]; then
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
-    args+=" -p $PWD/tests"
+    args+=" -p $test_folder"
 else
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
-    args+=" -s $PWD/tests/$sketch"
+    args+=" -s $test_folder/$sketch"
 fi
 
 ${BUILD_CMD} ${args} $*

.github/scripts/tests_run.sh

@@ -15,9 +15,9 @@ function run_test() {
     fi
 
     if [ $len -eq 1 ]; then
-      # build_dir="tests/$sketchname/build"
+      # build_dir="$sketchdir/build"
       build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
-      report_file="tests/$sketchname/$sketchname.xml"
+      report_file="$sketchdir/$sketchname.xml"
     fi
 
     for i in `seq 0 $(($len - 1))`
@@ -28,9 +28,9 @@ function run_test() {
         fi
 
         if [ $len -ne 1 ]; then
-            # build_dir="tests/$sketchname/build$i"
+            # build_dir="$sketchdir/build$i"
             build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
-            report_file="tests/$sketchname/$sketchname$i.xml"
+            report_file="$sketchdir/$sketchname$i.xml"
         fi
 
         pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file
@@ -79,6 +79,10 @@ while [ ! -z "$1" ]; do
         echo "$USAGE"
         exit 0
         ;;
+    -type )
+        shift
+        test_type=$1
+        ;;
     * )
       break
       ;;
@@ -88,8 +92,26 @@ done
 
 source ${SCRIPTS_DIR}/install-arduino-ide.sh
 
+# If sketch is provided and test type is not, test type is inferred from the sketch path
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+    if [ -n "$sketch" ]; then
+        tmp_sketch_path=$(find tests -name $sketch.ino)
+        test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+        echo "Sketch $sketch test type: $test_type"
+        test_folder="$PWD/tests/$test_type"
+    else
+      test_folder="$PWD/tests"
+    fi
+else
+    test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_run -eq 0 ]; then
-    run_test $target $PWD/tests/$sketch/$sketch.ino $options $erase
+    if [ -z $sketch ]; then
+        echo "ERROR: Sketch name is required for single test run"
+        return 1
+    fi
+    run_test $target $test_folder/$sketch/$sketch.ino $options $erase
 else
   if [ "$chunk_max" -le 0 ]; then
       echo "ERROR: Chunks count must be positive number"
@@ -102,7 +124,7 @@ else
   fi
 
   set +e
-  ${COUNT_SKETCHES} $PWD/tests $target
+  ${COUNT_SKETCHES} $test_folder $target
   sketchcount=$?
   set -e
   sketches=$(cat sketches.txt)