Refactor the CI scripts (espressif#6191)

The reason behind this refactoring is that all of the sketch related functions can (and will) be used for other purposes.

Build in the sketch directory: This will make it easy to handle artifacts after the build.
Separate sketch related functions from IDE installation script. This is the main commit.
Create a separate job for the Cmake check. This check was part of one of the Linux build. I believe that it's not the best place for such a check.
Checking for the skip landmarks and validity of the the sketch directory were already done by count_sketches.

Author: Ouss4

.github/scripts/install-arduino-ide.sh

@@ -34,9 +34,6 @@ else
 fi
 export OS_NAME
 
-ARDUINO_BUILD_DIR="$HOME/.arduino/build.tmp"
-ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
-
 if [ "$OS_IS_MACOS" == "1" ]; then
     export ARDUINO_IDE_PATH="/Applications/Arduino.app/Contents/Java"
     export ARDUINO_USR_PATH="$HOME/Documents/Arduino"
@@ -81,156 +78,3 @@ if [ ! -d "$ARDUINO_IDE_PATH" ]; then
     echo ""
 fi
 
-function build_sketch(){ # build_sketch <fqbn> <path-to-ino> [extra-options]
-    if [ "$#" -lt 2 ]; then
-        echo "ERROR: Illegal number of parameters"
-        echo "USAGE: build_sketch <fqbn> <path-to-ino> [extra-options]"
-        return 1
-    fi
-
-    local fqbn="$1"
-    local sketch="$2"
-    local xtra_opts="$3"
-    local win_opts=""
-    if [ "$OS_IS_WINDOWS" == "1" ]; then
-        local ctags_version=`ls "$ARDUINO_IDE_PATH/tools-builder/ctags/"`
-        local preprocessor_version=`ls "$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/"`
-        win_opts="-prefs=runtime.tools.ctags.path=$ARDUINO_IDE_PATH/tools-builder/ctags/$ctags_version -prefs=runtime.tools.arduino-preprocessor.path=$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/$preprocessor_version"
-    fi
-
-    #echo ""
-    #echo "Compiling '"$(basename "$sketch")"' ..."
-    mkdir -p "$ARDUINO_BUILD_DIR"
-    mkdir -p "$ARDUINO_CACHE_DIR"
-    $ARDUINO_IDE_PATH/arduino-builder -compile -logger=human -core-api-version=10810 \
-        -fqbn=$fqbn \
-        -warnings="all" \
-        -tools "$ARDUINO_IDE_PATH/tools-builder" \
-        -tools "$ARDUINO_IDE_PATH/tools" \
-        -built-in-libraries "$ARDUINO_IDE_PATH/libraries" \
-        -hardware "$ARDUINO_IDE_PATH/hardware" \
-        -hardware "$ARDUINO_USR_PATH/hardware" \
-        -libraries "$ARDUINO_USR_PATH/libraries" \
-        -build-cache "$ARDUINO_CACHE_DIR" \
-        -build-path "$ARDUINO_BUILD_DIR" \
-        $win_opts $xtra_opts "$sketch"
-}
-
-function count_sketches(){ # count_sketches <examples-path> <target-mcu>
-    local examples="$1"
-    local target="$2"
-    rm -rf sketches.txt
-    if [ ! -d "$examples" ]; then
-        touch sketches.txt
-        return 0
-    fi
-    local sketches=$(find $examples -name *.ino)
-    local sketchnum=0
-    for sketch in $sketches; do
-        local sketchdir=$(dirname $sketch)
-        local sketchdirname=$(basename $sketchdir)
-        local sketchname=$(basename $sketch)
-        if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
-            continue
-        elif [[ -f "$sketchdir/.skip.$target" ]]; then
-            continue
-        else
-            echo $sketch >> sketches.txt
-            sketchnum=$(($sketchnum + 1))
-        fi
-    done
-    return $sketchnum
-}
-
-function build_sketches(){ # build_sketches <fqbn> <target-mcu> <examples-path> <chunk> <total-chunks> [extra-options]
-    local fqbn=$1
-    local target="$2"
-    local examples=$3
-    local chunk_idex=$4
-    local chunks_num=$5
-    local xtra_opts=$6
-
-    if [ "$#" -lt 3 ]; then
-        echo "ERROR: Illegal number of parameters"
-        echo "USAGE: build_sketches <fqbn> <target-mcu <examples-path> [<chunk> <total-chunks>] [extra-options]"
-        return 1
-    fi
-
-    if [ "$#" -lt 5 ]; then
-        chunk_idex="0"
-        chunks_num="1"
-        xtra_opts=$4
-    fi
-
-    if [ "$chunks_num" -le 0 ]; then
-        echo "ERROR: Chunks count must be positive number"
-        return 1
-    fi
-    if [ "$chunk_idex" -ge "$chunks_num" ] && [ "$chunks_num" -ge 2 ]; then
-        echo "ERROR: Chunk index must be less than chunks count"
-        return 1
-    fi
-
-    set +e
-    count_sketches "$examples" "$target"
-    local sketchcount=$?
-    set -e
-    local sketches=$(cat sketches.txt)
-    rm -rf sketches.txt
-
-    local chunk_size=$(( $sketchcount / $chunks_num ))
-    local all_chunks=$(( $chunks_num * $chunk_size ))
-    if [ "$all_chunks" -lt "$sketchcount" ]; then
-        chunk_size=$(( $chunk_size + 1 ))
-    fi
-
-    local start_index=0
-    local end_index=0
-    if [ "$chunk_idex" -ge "$chunks_num" ]; then
-        start_index=$chunk_idex
-        end_index=$sketchcount
-    else
-        start_index=$(( $chunk_idex * $chunk_size ))
-        if [ "$sketchcount" -le "$start_index" ]; then
-            echo "Skipping job"
-            return 0
-        fi
-
-        end_index=$(( $(( $chunk_idex + 1 )) * $chunk_size ))
-        if [ "$end_index" -gt "$sketchcount" ]; then
-            end_index=$sketchcount
-        fi
-    fi
-
-    local start_num=$(( $start_index + 1 ))
-    echo "Found $sketchcount Sketches for target '$target'";
-    echo "Chunk Index : $chunk_idex"
-    echo "Chunk Count : $chunks_num"
-    echo "Chunk Size  : $chunk_size"
-    echo "Start Sketch: $start_num"
-    echo "End Sketch  : $end_index"
-
-    local sketchnum=0
-    for sketch in $sketches; do
-        local sketchdir=$(dirname $sketch)
-        local sketchdirname=$(basename $sketchdir)
-        local sketchname=$(basename $sketch)
-        if [ "${sketchdirname}.ino" != "$sketchname" ] \
-        || [ -f "$sketchdir/.skip.$target" ]; then
-            continue
-        fi
-        sketchnum=$(($sketchnum + 1))
-        if [ "$sketchnum" -le "$start_index" ] \
-        || [ "$sketchnum" -gt "$end_index" ]; then
-            continue
-        fi
-        echo ""
-        echo "Building Sketch Index $(($sketchnum - 1)) - $sketchdirname"
-        build_sketch "$fqbn" "$sketch" "$xtra_opts"
-        local result=$?
-        if [ $result -ne 0 ]; then
-            return $result
-        fi
-    done
-    return 0
-}

.github/scripts/on-push.sh

@@ -2,6 +2,40 @@
 
 set -e
 
+function build(){
+    local target=$1
+    local fqbn=$2
+    local chunk_index=$3
+    local chunks_cnt=$4
+    local sketches=$5
+
+    local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
+    local BUILD_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
+
+    local args="$ARDUINO_IDE_PATH $ARDUINO_USR_PATH"
+
+    args+=" \"$fqbn\""
+
+    if [ "$OS_IS_LINUX" == "1" ]; then
+        args+=" $target"
+        args+=" $ARDUINO_ESP32_PATH/libraries"
+        args+=" $chunk_index $chunks_cnt"
+        ${BUILD_SKETCHES} ${args}
+    else
+        if [ "$OS_IS_WINDOWS" == "1" ]; then
+            local ctags_version=`ls "$ARDUINO_IDE_PATH/tools-builder/ctags/"`
+            local preprocessor_version=`ls "$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/"`
+            win_opts="-prefs=runtime.tools.ctags.path=$ARDUINO_IDE_PATH/tools-builder/ctags/$ctags_version
+            -prefs=runtime.tools.arduino-preprocessor.path=$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/$preprocessor_version"
+            args+=" ${win_opts}"
+        fi
+
+        for sketch in ${sketches}; do
+            ${BUILD_SKETCH} ${args} ${sketch}
+        done
+    fi
+}
+
 if [ -z "$GITHUB_WORKSPACE" ]; then
     export GITHUB_WORKSPACE="$PWD"
     export GITHUB_REPOSITORY="espressif/arduino-esp32"
@@ -22,57 +56,31 @@ fi
 #echo "Updating submodules ..."
 #git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1
 
+SCRIPTS_DIR="./.github/scripts"
 if [ "$BUILD_PIO" -eq 0 ]; then
-    # ArduinoIDE ESP32 Test
-    TARGET="esp32"
-    FQBN="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
     source ./.github/scripts/install-arduino-ide.sh
-    source ./.github/scripts/install-arduino-core-esp32.sh
-    if [ "$OS_IS_WINDOWS" == "1" ]; then
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
-    elif [ "$OS_IS_MACOS" == "1" ]; then
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
-    else
-        # CMake Test
-        if [ "$CHUNK_INDEX" -eq 0 ]; then
-            bash "$ARDUINO_ESP32_PATH/.github/scripts/check-cmakelists.sh"
-        fi
-        build_sketches "$FQBN" "$TARGET" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
-    fi
+    source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
-    # ArduinoIDE ESP32S2 Test
-    TARGET="esp32s2"
-    FQBN="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
-    if [ "$OS_IS_WINDOWS" == "1" ]; then
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
-    elif [ "$OS_IS_MACOS" == "1" ]; then
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
-    else
-        build_sketches "$FQBN" "$TARGET" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
-    fi
+    FQBN_ESP32="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
+    FQBN_ESP32S2="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
+    FQBN_ESP32C3="espressif:esp32:esp32c3:PartitionScheme=huge_app"
 
-    # ArduinoIDE ESP32C3 Test
-    TARGET="esp32c3"
-    FQBN="espressif:esp32:esp32c3:PartitionScheme=huge_app"
-    if [ "$OS_IS_WINDOWS" == "1" ]; then
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
-    elif [ "$OS_IS_MACOS" == "1" ]; then
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
-        build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
-    else
-        build_sketches "$FQBN" "$TARGET" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
-    fi
+    SKETCHES_ESP32="\
+      $ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino\
+      $ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino\
+      $ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino\
+    "
+
+    SKETCHES_ESP32XX="\
+      $ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino\
+      $ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino\
+    "
+
+    build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
+    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
+    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
 else
-    source ./.github/scripts/install-platformio-esp32.sh
+    source ./${SCRIPTS_DIR}/install-platformio-esp32.sh
     # PlatformIO ESP32 Test
     BOARD="esp32dev"
     OPTIONS="board_build.partitions = huge_app.csv"

.github/scripts/sketch_utils.sh

@@ -0,0 +1,184 @@
+#!/bin/bash
+
+function build_sketch(){ # build_sketch <ide_path> <user_path> <fqbn> <path-to-ino> [extra-options]
+    if [ "$#" -lt 4 ]; then
+        echo "ERROR: Illegal number of parameters"
+        echo "USAGE: ${0} build <ide_path> <user_path> <fqbn> <path-to-ino> [extra-options]"
+        return 1
+    fi
+
+    ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
+    local ide_path=$1
+    local usr_path=$2
+    local fqbn=$3
+    local sketch=$4
+    local xtra_opts=$5
+    local win_opts=$6
+
+    build_dir="$(dirname $sketch)/build"
+    mkdir -p "$build_dir"
+    mkdir -p "$ARDUINO_CACHE_DIR"
+    $ide_path/arduino-builder -compile -logger=human -core-api-version=10810 \
+        -fqbn=$fqbn \
+        -warnings="all" \
+        -tools "$ide_path/tools-builder" \
+        -tools "$ide_path/tools" \
+        -built-in-libraries "$ide_path/libraries" \
+        -hardware "$ide_path/hardware" \
+        -hardware "$usr_path/hardware" \
+        -libraries "$usr_path/libraries" \
+        -build-cache "$ARDUINO_CACHE_DIR" \
+        -build-path "$build_dir" \
+        $win_opts $xtra_opts "$sketch"
+}
+
+function count_sketches(){ # count_sketches <path> <target>
+    local path=$1
+    local target=$2
+
+    if [ $# -lt 2 ]; then
+      echo "ERROR: Illegal number of parameters"
+      echo "USAGE: ${0} count <path> <target>"
+    fi
+
+    rm -rf sketches.txt
+    if [ ! -d "$path" ]; then
+        touch sketches.txt
+        return 0
+    fi
+
+    local sketches=$(find $path -name *.ino)
+    local sketchnum=0
+    for sketch in $sketches; do
+        local sketchdir=$(dirname $sketch)
+        local sketchdirname=$(basename $sketchdir)
+        local sketchname=$(basename $sketch)
+        if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
+            continue
+        elif [[ -f "$sketchdir/.skip.$target" ]]; then
+            continue
+        else
+            echo $sketch >> sketches.txt
+            sketchnum=$(($sketchnum + 1))
+        fi
+    done
+    return $sketchnum
+}
+
+function build_sketches(){ # build_sketches <ide_path> <user_path> <fqbn> <target> <path> <chunk> <total-chunks> [extra-options]
+    local ide_path=$1
+    local usr_path=$2
+    local fqbn=$3
+    local target=$4
+    local path=$5
+    local chunk_idex=$6
+    local chunks_num=$7
+    local xtra_opts=$8
+
+    if [ "$#" -lt 7 ]; then
+        echo "ERROR: Illegal number of parameters"
+        echo "USAGE: ${0} chunk_build <ide_path> <user_path> <fqbn> <target> <path> [<chunk> <total-chunks>] [extra-options]"
+        return 1
+    fi
+
+    if [ "$chunks_num" -le 0 ]; then
+        echo "ERROR: Chunks count must be positive number"
+        return 1
+    fi
+    if [ "$chunk_idex" -ge "$chunks_num" ] && [ "$chunks_num" -ge 2 ]; then
+        echo "ERROR: Chunk index must be less than chunks count"
+        return 1
+    fi
+
+    set +e
+    count_sketches "$path" "$target"
+    local sketchcount=$?
+    set -e
+    local sketches=$(cat sketches.txt)
+    rm -rf sketches.txt
+
+    local chunk_size=$(( $sketchcount / $chunks_num ))
+    local all_chunks=$(( $chunks_num * $chunk_size ))
+    if [ "$all_chunks" -lt "$sketchcount" ]; then
+        chunk_size=$(( $chunk_size + 1 ))
+    fi
+
+    local start_index=0
+    local end_index=0
+    if [ "$chunk_idex" -ge "$chunks_num" ]; then
+        start_index=$chunk_idex
+        end_index=$sketchcount
+    else
+        start_index=$(( $chunk_idex * $chunk_size ))
+        if [ "$sketchcount" -le "$start_index" ]; then
+            echo "Skipping job"
+            return 0
+        fi
+
+        end_index=$(( $(( $chunk_idex + 1 )) * $chunk_size ))
+        if [ "$end_index" -gt "$sketchcount" ]; then
+            end_index=$sketchcount
+        fi
+    fi
+
+    local start_num=$(( $start_index + 1 ))
+    echo "Found $sketchcount Sketches for target '$target'";
+    echo "Chunk Index : $chunk_idex"
+    echo "Chunk Count : $chunks_num"
+    echo "Chunk Size  : $chunk_size"
+    echo "Start Sketch: $start_num"
+    echo "End Sketch  : $end_index"
+
+    local sketchnum=0
+    for sketch in $sketches; do
+        local sketchdir=$(dirname $sketch)
+        local sketchdirname=$(basename $sketchdir)
+        local sketchname=$(basename $sketch)
+        sketchnum=$(($sketchnum + 1))
+        if [ "$sketchnum" -le "$start_index" ] \
+        || [ "$sketchnum" -gt "$end_index" ]; then
+            continue
+        fi
+        echo ""
+        echo "Building Sketch Index $(($sketchnum - 1)) - $sketchdirname"
+        build_sketch "$ide_path" "$usr_path" "$fqbn" "$sketch" "$xtra_opts"
+        local result=$?
+        if [ $result -ne 0 ]; then
+            return $result
+        fi
+    done
+    return 0
+}
+
+USAGE="
+USAGE: ${0} [command] [options]
+Available commands:
+  count: Count sketches.
+  build: Build a sketch.
+  chunk_build: Build a chunk of sketches.
+"
+
+cmd=$1
+shift
+if [ -z $cmd ]; then
+  echo "ERROR: No command supplied"
+  echo "$USAGE"
+  exit 2
+fi
+
+case "$cmd" in
+  "count")
+    count_sketches $*
+    ;;
+  "build")
+    build_sketch $*
+    ;;
+  "chunk_build")
+    build_sketches $*
+    ;;
+  *)
+    echo "ERROR: Unrecognized command"
+    echo "$USAGE"
+    exit 2
+esac
+

.github/workflows/push.yml

@@ -14,6 +14,13 @@ concurrency:
 
 jobs:
 
+  cmake-check:
+    name: Check cmake file
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: bash ./.github/scripts/check-cmakelists.sh
+
   # Ubuntu
   build-arduino-linux:
     name: Arduino ${{ matrix.chunk }} on ubuntu-latest