Merge branch 'master' into nb1

Author: earlephilhower

Diff for: .flake8

@@ -1,12 +1,10 @@
 # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-python/.flake8
 # See: https://flake8.pycqa.org/en/latest/user/configuration.html
-# The code style defined in this file is the official standardized style to be used in all Arduino tooling projects and
-# should not be modified.
 
 [flake8]
 doctests = True
 # W503 and W504 are mutually exclusive. PEP 8 recommends line break before.
 ignore = W503,E203
-max-complexity = 10
+max-complexity = 20
 max-line-length = 120
 select = E,W,F,C,N

Diff for: .github/ISSUE_TEMPLATE/Issue-report.yml

@@ -41,6 +41,11 @@ body:
       options:
         - latest master (checkout manually)
         - latest development Release Candidate (RC-X)
+        - v3.0.1
+        - v3.0.0
+        - v2.0.17
+        - v2.0.16
+        - v2.0.15
         - v2.0.14
         - v2.0.13
         - v2.0.12

Diff for: .github/scripts/find_new_boards.sh

@@ -54,7 +54,7 @@ do
     break
     fi
     board_name=$(echo "$line" | cut -d '.' -f1 | cut -d '#' -f1)
-    if [ "$board_name" != "" ]
+    if [ "$board_name" != "" ] && [ "$board_name" != "esp32_family" ]
     then
         if [ "$board_name" != "$previous_board" ]
         then

Diff for: .github/scripts/install-platformio-esp32.sh

@@ -89,7 +89,9 @@ function count_sketches(){ # count_sketches <examples-path>
         if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
             continue
         fi
-        if [[ -f "$sketchdir/.test.skip" ]]; then
+        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
+        # If the target is listed as false, skip the sketch. Otherwise, include it.
+        if [[ "$is_target" == "false" ]]; then
             continue
         fi
         echo $sketch >> sketches.txt
@@ -161,8 +163,10 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
         local sketchname=$(basename $sketch)
+        is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
+        # If the target is listed as false, skip the sketch. Otherwise, include it.
         if [ "${sketchdirname}.ino" != "$sketchname" ] \
-        || [ -f "$sketchdir/.test.skip" ]; then
+        || [[ "$is_target" == "false" ]]; then
             continue
         fi
         sketchnum=$(($sketchnum + 1))

Diff for: .github/scripts/merge_packages.py

@@ -5,7 +5,8 @@
 # Written by Ivan Grokhotkov, 2015
 #
 from __future__ import print_function
-from distutils.version import LooseVersion
+#from distutils.version import LooseVersion
+from packaging.version import Version
 import re
 import json
 import sys
@@ -33,7 +34,7 @@ def merge_objects(versions, obj):
 def pkgVersionNormalized(versionString):
 
     verStr = str(versionString)
-    verParts = re.split('\.|-rc', verStr, flags=re.IGNORECASE)
+    verParts = re.split('\.|-rc|-alpha', verStr, flags=re.IGNORECASE)
 
     if len(verParts) == 3:
         if (sys.version_info > (3, 0)): # Python 3
@@ -74,7 +75,8 @@ def main(args):
             print("Adding platform {0}-{1}".format(name, version), file=sys.stderr)
             pkg1['platforms'].append(platforms[name][version])
 
-    pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: LooseVersion(pkgVersionNormalized(k['version'])), reverse=True)
+    #pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: LooseVersion(pkgVersionNormalized(k['version'])), reverse=True)
+    pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: Version(pkgVersionNormalized(k['version'])), reverse=True)
 
     json.dump({'packages':[pkg1]}, sys.stdout, indent=2)
 

Diff for: .github/scripts/on-push.sh

@@ -10,7 +10,8 @@ function build(){
     local chunk_index=$3
     local chunks_cnt=$4
     local build_log=$5
-    shift; shift; shift; shift; shift;
+    local sketches_file=$6
+    shift; shift; shift; shift; shift; shift;
     local sketches=$*
 
     local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
@@ -23,6 +24,9 @@ function build(){
     if [ "$OS_IS_LINUX" == "1" ]; then
         args+=" -p $ARDUINO_ESP32_PATH/libraries"
         args+=" -i $chunk_index -m $chunks_cnt"
+        if [ -n "$sketches_file" ]; then
+          args+=" -f $sketches_file"
+        fi
         if [ $build_log -eq 1 ]; then
             args+=" -l $build_log"
         fi
@@ -50,6 +54,7 @@ fi
 CHUNK_INDEX=$1
 CHUNKS_CNT=$2
 BUILD_LOG=$3
+SKETCHES_FILE=$4
 BUILD_PIO=0
 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then
     CHUNK_INDEX=0
@@ -60,7 +65,7 @@ elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then
     BUILD_PIO=1
 fi
 
-if [ "$BUILD_LOG" -le 0 ]; then
+if [ -z "$BUILD_LOG" ] || [ "$BUILD_LOG" -le 0 ]; then
     BUILD_LOG=0
 fi
 
@@ -69,7 +74,6 @@ fi
 
 SCRIPTS_DIR="./.github/scripts"
 if [ "$BUILD_PIO" -eq 0 ]; then
-    #source ${SCRIPTS_DIR}/install-arduino-ide.sh
     source ${SCRIPTS_DIR}/install-arduino-cli.sh
     source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
@@ -95,16 +99,16 @@ if [ "$BUILD_PIO" -eq 0 ]; then
     fi
 
     #build sketches for different targets
-    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32c6" $FQBN_ESP32C6 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32h2" $FQBN_ESP32H2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
-    build "esp32"   $FQBN_ESP32   $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
+    build "esp32s3" $FQBN_ESP32S3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32s2" $FQBN_ESP32S2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32c3" $FQBN_ESP32C3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32c6" $FQBN_ESP32C6 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32h2" $FQBN_ESP32H2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
+    build "esp32"   $FQBN_ESP32   "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
 
     if [ "$BUILD_LOG" -eq 1 ]; then
         #remove last comma from the last JSON object
-        sed -i '$ s/.$//' "$sizes_file"
+        sed -i '$ s/,$//' "$sizes_file"
         #echo end of JSON array
         echo "]}" >> $sizes_file
     fi

Diff for: .github/scripts/on-release.sh

@@ -216,16 +216,23 @@ find "$PKG_DIR" -name '*.git*' -type f -delete
 ##
 RVTC_NAME="riscv32-esp-elf-gcc"
 RVTC_NEW_NAME="esp-rv32"
+X32TC_NAME="xtensa-esp32-elf-gcc"
+X32TC_NEW_NAME="esp-x32"
+XS2TC_NAME="xtensa-esp32s2-elf-gcc"
+XS2TC_NEW_NAME="esp-xs2"
+XS3TC_NAME="xtensa-esp32s3-elf-gcc"
+XS3TC_NEW_NAME="esp-xs3"
 
 # Replace tools locations in platform.txt
 echo "Generating platform.txt..."
 cat "$GITHUB_WORKSPACE/platform.txt" | \
 sed "s/version=.*/version=$RELEASE_TAG/g" | \
+sed 's/tools\.esp32-arduino-libs\.path\.windows=.*//g' | \
 sed 's/{runtime\.platform\.path}.tools.esp32-arduino-libs/\{runtime.tools.esp32-arduino-libs.path\}/g' | \
 sed 's/{runtime\.platform\.path}.tools.xtensa-esp-elf-gdb/\{runtime.tools.xtensa-esp-elf-gdb.path\}/g' | \
-sed 's/{runtime\.platform\.path}.tools.xtensa-esp32-elf/\{runtime.tools.xtensa-esp32-elf-gcc.path\}/g' | \
-sed 's/{runtime\.platform\.path}.tools.xtensa-esp32s2-elf/\{runtime.tools.xtensa-esp32s2-elf-gcc.path\}/g' | \
-sed 's/{runtime\.platform\.path}.tools.xtensa-esp32s3-elf/\{runtime.tools.xtensa-esp32s3-elf-gcc.path\}/g' | \
+sed "s/{runtime\.platform\.path}.tools.xtensa-esp32-elf/\\{runtime.tools.$X32TC_NEW_NAME.path\\}/g" | \
+sed "s/{runtime\.platform\.path}.tools.xtensa-esp32s2-elf/\\{runtime.tools.$XS2TC_NEW_NAME.path\\}/g" | \
+sed "s/{runtime\.platform\.path}.tools.xtensa-esp32s3-elf/\\{runtime.tools.$XS3TC_NEW_NAME.path\\}/g" | \
 sed 's/{runtime\.platform\.path}.tools.riscv32-esp-elf-gdb/\{runtime.tools.riscv32-esp-elf-gdb.path\}/g' | \
 sed "s/{runtime\.platform\.path}.tools.riscv32-esp-elf/\\{runtime.tools.$RVTC_NEW_NAME.path\\}/g" | \
 sed 's/{runtime\.platform\.path}.tools.esptool/\{runtime.tools.esptool_py.path\}/g' | \
@@ -341,7 +348,19 @@ rvtc_jq_arg="\
     (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$RVTC_NAME\")).version = \"$RVTC_VERSION\" |\
     (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$RVTC_NAME\")).name = \"$RVTC_NEW_NAME\" |\
     (.packages[0].tools[] | select(.name==\"$RVTC_NAME\")).version = \"$RVTC_VERSION\" |\
-    (.packages[0].tools[] | select(.name==\"$RVTC_NAME\")).name = \"$RVTC_NEW_NAME\""
+    (.packages[0].tools[] | select(.name==\"$RVTC_NAME\")).name = \"$RVTC_NEW_NAME\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$X32TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$X32TC_NAME\")).name = \"$X32TC_NEW_NAME\" |\
+    (.packages[0].tools[] | select(.name==\"$X32TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].tools[] | select(.name==\"$X32TC_NAME\")).name = \"$X32TC_NEW_NAME\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$XS2TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$XS2TC_NAME\")).name = \"$XS2TC_NEW_NAME\" |\
+    (.packages[0].tools[] | select(.name==\"$XS2TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].tools[] | select(.name==\"$XS2TC_NAME\")).name = \"$XS2TC_NEW_NAME\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$XS3TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$XS3TC_NAME\")).name = \"$XS3TC_NEW_NAME\" |\
+    (.packages[0].tools[] | select(.name==\"$XS3TC_NAME\")).version = \"$RVTC_VERSION\" |\
+    (.packages[0].tools[] | select(.name==\"$XS3TC_NAME\")).name = \"$XS3TC_NEW_NAME\""
 cat "$PACKAGE_JSON_TEMPLATE" | jq "$rvtc_jq_arg" > "$OUTPUT_DIR/package-$LIBS_PROJ_NAME-rvfix.json"
 PACKAGE_JSON_TEMPLATE="$OUTPUT_DIR/package-$LIBS_PROJ_NAME-rvfix.json"
 

Diff for: .github/scripts/set_push_chunks.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+build_all=false
+chunks_count=0
+
+if [[ $CORE_CHANGED == 'true' ]] || [[ $IS_PR != 'true' ]]; then
+  echo "Core files changed or not a PR. Building all."
+  build_all=true
+  chunks_count=$MAX_CHUNKS
+elif [[ $LIB_CHANGED == 'true' ]]; then
+  echo "Libraries changed. Building only affected sketches."
+  if [[ $NETWORKING_CHANGED == 'true' ]]; then
+    echo "Networking libraries changed. Building networking related sketches."
+    networking_sketches="$(find libraries/WiFi -name *.ino) "
+    networking_sketches+="$(find libraries/Ethernet -name *.ino) "
+    networking_sketches+="$(find libraries/PPP -name *.ino) "
+    networking_sketches+="$(find libraries/NetworkClientSecure -name *.ino) "
+    networking_sketches+="$(find libraries/WebServer -name *.ino) "
+  fi
+  if [[ $FS_CHANGED == 'true' ]]; then
+    echo "FS libraries changed. Building FS related sketches."
+    fs_sketches="$(find libraries/SD -name *.ino) "
+    fs_sketches+="$(find libraries/SD_MMC -name *.ino) "
+    fs_sketches+="$(find libraries/SPIFFS -name *.ino) "
+    fs_sketches+="$(find libraries/LittleFS -name *.ino) "
+    fs_sketches+="$(find libraries/FFat -name *.ino) "
+  fi
+  sketches="$networking_sketches $fs_sketches"
+  for file in $LIB_FILES; do
+      if [[ $file == *.ino ]]; then
+          # If file ends with .ino, add it to the list of sketches
+          echo "Sketch found: $file"
+          sketches+="$file "
+      elif [[ $(basename $(dirname $file)) == "src" ]]; then
+          # If file is in a src directory, find all sketches in the parent/examples directory
+          echo "Library src file found: $file"
+          lib=$(dirname $(dirname $file))
+          if [[ -d $lib/examples ]]; then
+              lib_sketches=$(find $lib/examples -name *.ino)
+              sketches+="$lib_sketches "
+              echo "Library sketches: $lib_sketches"
+          fi
+      else
+          # If file is in a example folder but it is not a sketch, find all sketches in the current directory
+          echo "File in example folder found: $file"
+          sketch=$(find $(dirname $file) -name *.ino)
+          sketches+="$sketch "
+          echo "Sketch in example folder: $sketch"
+      fi
+      echo ""
+  done
+fi
+
+if [[ -n $sketches ]]; then
+  # Remove duplicates
+  sketches=$(echo $sketches | tr ' ' '\n' | sort | uniq)
+  for sketch in $sketches; do
+      echo $sketch >> sketches_found.txt
+      chunks_count=$((chunks_count+1))
+  done
+  echo "Number of sketches found: $chunks_count"
+  echo "Sketches:"
+  echo "$sketches"
+
+  if [[ $chunks_count -gt $MAX_CHUNKS ]]; then
+    echo "More sketches than the allowed number of chunks found. Limiting to $MAX_CHUNKS chunks."
+    chunks_count=$MAX_CHUNKS
+  fi
+fi
+
+chunks='["0"'
+for i in $(seq 1 $(( $chunks_count - 1 )) ); do
+  chunks+=",\"$i\""
+done
+chunks+="]"
+
+echo "build_all=$build_all" >> $GITHUB_OUTPUT
+echo "build_libraries=$BUILD_LIBRARIES" >> $GITHUB_OUTPUT
+echo "build_static_sketches=$BUILD_STATIC_SKETCHES" >> $GITHUB_OUTPUT
+echo "build_idf=$BUILD_IDF" >> $GITHUB_OUTPUT
+echo "build_platformio=$BUILD_PLATFORMIO" >> $GITHUB_OUTPUT
+echo "chunk_count=$chunks_count" >> $GITHUB_OUTPUT
+echo "chunks=$chunks" >> $GITHUB_OUTPUT