ci(qemu): Add QEMU emulator to CI

Author: lucasssvaz

.github/scripts/sketch_utils.sh

@@ -78,10 +78,10 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
 
             # Default FQBN options if none were passed in the command line.
 
-            esp32_opts="PSRAM=enabled,PartitionScheme=huge_app"
+            esp32_opts="FlashMode=dio,PSRAM=enabled,PartitionScheme=huge_app"
             esp32s2_opts="PSRAM=enabled,PartitionScheme=huge_app"
             esp32s3_opts="PSRAM=opi,USBMode=default,PartitionScheme=huge_app"
-            esp32c3_opts="PartitionScheme=huge_app"
+            esp32c3_opts="FlashMode=dio,PartitionScheme=huge_app"
             esp32c6_opts="PartitionScheme=huge_app"
             esp32h2_opts="PartitionScheme=huge_app"
 
@@ -139,7 +139,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
         echo "Skipping $sketchname for target $target"
         exit 0
     fi
-    
+
     ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
     if [ -n "$ARDUINO_BUILD_DIR" ]; then
         build_dir="$ARDUINO_BUILD_DIR"
@@ -177,7 +177,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 --build-path "$build_dir" \
                 $xtra_opts "${sketchdir}" \
                 > $output_file
-            
+
             exit_status=$?
             if [ $exit_status -ne 0 ]; then
                 echo ""ERROR: Compilation failed with error code $exit_status""
@@ -198,11 +198,11 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 # Extract the desired substring using sed
                 lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||")
                 #append json file where key is fqbn, sketch name, sizes -> extracted values
-                echo "{\"name\": \"$lib_sketch_name\", 
+                echo "{\"name\": \"$lib_sketch_name\",
                     \"sizes\": [{
-                            \"flash_bytes\": $flash_bytes, 
-                            \"flash_percentage\": $flash_percentage, 
-                            \"ram_bytes\": $ram_bytes, 
+                            \"flash_bytes\": $flash_bytes,
+                            \"flash_percentage\": $flash_percentage,
+                            \"ram_bytes\": $ram_bytes,
                             \"ram_percentage\": $ram_percentage
                             }]
                     }," >> "$sizes_file"
@@ -386,7 +386,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
     if [ $log_compilation ]; then
         #echo board,target and start of sketches to sizes_file json
         echo "{ \"board\": \"$fqbn\",
-                \"target\": \"$target\", 
+                \"target\": \"$target\",
                 \"sketches\": [" >> "$sizes_file"
     fi
 

.github/scripts/tests_run.sh

@@ -39,19 +39,23 @@ function run_test() {
         fi
 
         if [ $platform == "qemu" ]; then
-            extra_args="--embedded-services qemu --qemu-image-path $build_dir/$sketchname.ino.bin"
+            PATH=$HOME/qemu/bin:$PATH
+            extra_args="--embedded-services qemu --qemu-image-path $build_dir/$sketchname.ino.merged.bin"
 
             if [ $target == "esp32" ] || [ $target == "esp32s3" ]; then
-                extra_args+=" --qemu-prog-path $QEMU_PATH/qemu-system-xtensa --qemu-cli-args '-nographic -machine $target'"
+                extra_args+=" --qemu-prog-path qemu-system-xtensa --qemu-cli-args=\"-machine $target -nographic\""
+            elif [ $target == "esp32c3" ]; then
+                extra_args+=" --qemu-prog-path qemu-system-riscv32 --qemu-cli-args=\"-machine $target -icount 3 -nographic\""
             else
-                extra_args+=" --qemu-prog-path $QEMU_PATH/qemu-system-riscv32 --qemu-cli-args '-nographic -machine $target'"
+                echo "Unsupported QEMU target: $target"
+                exit 1
             fi
         else
             extra_args="--embedded-services esp,arduino"
         fi
 
         echo "pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file $extra_args"
-        pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file $extra_args
+        bash -c "pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file $extra_args"
         result=$?
         if [ $result -ne 0 ]; then
             return $result

.github/workflows/hil.yml

@@ -95,10 +95,11 @@ jobs:
             ~/.cache/pip
           key: ${{ steps.get-qemu-version.outputs.release }}-${{ hashFiles('.github/workflows/hil.yml') }}
 
-      - name: Install python dependencies
+      - name: Install dependencies
         run: |
           pip install -U pip
           pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
+          sudo apt update && sudo apt install libpixman-1-0 libnuma1 libglib2.0-0 libslirp0 libsdl2-2.0-0
 
       - name: Download QEMU
         if: steps.cache-linux.outputs.cache-hit != 'true'

platform.txt

@@ -169,6 +169,17 @@ recipe.hooks.objcopy.postobjcopy.1.pattern.windows=cmd /c if exist "{build.path}
 recipe.hooks.objcopy.postobjcopy.2.pattern=/usr/bin/env bash -c "[ ! -d "{build.path}"/libraries/ESP_SR ] || [ ! -f "{compiler.sdk.path}"/esp_sr/srmodels.bin ] || cp -f "{compiler.sdk.path}"/esp_sr/srmodels.bin "{build.path}"/srmodels.bin"
 recipe.hooks.objcopy.postobjcopy.2.pattern.windows=cmd /c if exist "{build.path}\libraries\ESP_SR" if exist "{compiler.sdk.path}\esp_sr\srmodels.bin" COPY /y "{compiler.sdk.path}\esp_sr\srmodels.bin" "{build.path}\srmodels.bin"
 
+## Create flash_args file
+flash_args.path={build.path}/flash_args
+recipe.hooks.objcopy.postobjcopy.3.pattern_args=(echo '--fill-flash-size {build.flash_size} --flash_mode keep --flash_freq keep --flash_size keep'; echo '{build.bootloader_addr} {build.path}/{build.project_name}.bootloader.bin'; echo '0x8000 {build.path}/{build.project_name}.partitions.bin'; echo '0xe000 {runtime.platform.path}/tools/partitions/boot_app0.bin'; echo '0x10000 {build.path}/{build.project_name}.bin') > {flash_args.path}
+recipe.hooks.objcopy.postobjcopy.3.pattern=/usr/bin/env bash -c "{recipe.hooks.objcopy.postobjcopy.3.pattern_args}"
+recipe.hooks.objcopy.postobjcopy.3.pattern.windows=cmd /c "{recipe.hooks.objcopy.postobjcopy.3.pattern_args}"
+
+# Create QEMU image
+recipe.hooks.objcopy.postobjcopy.4.pattern_args=--chip {build.mcu} merge_bin -o {build.path}/{build.project_name}.merged.bin @{flash_args.path}
+recipe.hooks.objcopy.postobjcopy.4.pattern="{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.hooks.objcopy.postobjcopy.4.pattern_args}
+recipe.hooks.objcopy.postobjcopy.4.pattern.linux=python3 "{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.hooks.objcopy.postobjcopy.4.pattern_args}
+
 ## Save bin
 recipe.output.tmp_file={build.project_name}.bin
 recipe.output.save_file={build.project_name}.{build.variant}.bin

tests/democfg/.skip.qemu

@@ -1,6 +1,6 @@
 cryptography>=2.1.4
 --only-binary cryptography
 pytest-cov
-pytest-embedded-serial-esp>=1.3.4
-pytest-embedded-arduino>=1.3.4
-pytest-embedded-qemu>=1.3.4
+pytest-embedded-serial-esp>=1.10.1
+pytest-embedded-arduino>=1.10.1
+pytest-embedded-qemu>=1.10.1