Fix/changed files

.github/ISSUE_TEMPLATE/Issue-report.yml

@@ -41,6 +41,8 @@ body:
       options:
         - latest master (checkout manually)
         - latest development Release Candidate (RC-X)
+        - v3.1.2
+        - v3.1.1
         - v3.1.0
         - v3.0.7
         - v3.0.6

.github/scripts/set_push_chunks.sh

@@ -2,6 +2,78 @@
 
 build_all=false
 chunks_count=0
+last_check_files=""
+
+# Define the file patterns
+core_files=(
+    '\.github/.*'
+    'cores/.*'
+    'package/.*'
+    'tools/.*'
+    'platform\.txt'
+    'programmers\.txt'
+    'variants/esp32/.*'
+    'variants/esp32c3/.*'
+    'variants/esp32c6/.*'
+    'variants/esp32h2/.*'
+    'variants/esp32p4/.*'
+    'variants/esp32s2/.*'
+    'variants/esp32s3/.*'
+)
+library_files=(
+    'libraries/.*/examples/.*'
+    'libraries/.*/src/.*'
+)
+networking_files=(
+    'libraries/Network/src/.*'
+)
+fs_files=(
+    'libraries/FS/src/.*'
+)
+static_sketches_files=(
+    'libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure\.ino'
+    'libraries/BLE/examples/Server/Server\.ino'
+    'libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer\.ino'
+    'libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics\.ino'
+    'libraries/NetworkClientSecure/src/.*'
+    'libraries/BLE/src/.*'
+    'libraries/Insights/src/.*'
+)
+idf_files=(
+    'idf_component\.yml'
+    'Kconfig\.projbuild'
+    'CMakeLists\.txt'
+    'variants/esp32c2/.*'
+)
+
+# Function to check if any files match the patterns
+check_files() {
+    local patterns=("$@")
+    local files_found=""
+    for pattern in "${patterns[@]}"; do
+        if [[ $IS_PR != 'true' ]]; then
+            files_found+="$(gh api repos/espressif/arduino-esp32/commits/"$GITHUB_SHA" --jq '.files[].filename' | grep -E "$pattern") "
+        else
+            files_found+="$(gh pr diff "$PR_NUM" --name-only | grep -E "$pattern") "
+        fi
+    done
+
+    last_check_files=$(echo "$files_found" | xargs)
+    if [[ -n $last_check_files ]]; then
+        echo 'true'
+    else
+        echo 'false'
+    fi
+}
+
+# Output the results
+CORE_CHANGED=$(check_files "${core_files[@]}")
+LIB_CHANGED=$(check_files "${library_files[@]}")
+LIB_FILES=$last_check_files
+NETWORKING_CHANGED=$(check_files "${networking_files[@]}")
+FS_CHANGED=$(check_files "${fs_files[@]}")
+STATIC_SKETCHES_CHANGED=$(check_files "${static_sketches_files[@]}")
+IDF_CHANGED=$(check_files "${idf_files[@]}")
 
 if [[ $CORE_CHANGED == 'true' ]] || [[ $IS_PR != 'true' ]]; then
     echo "Core files changed or not a PR. Building all."
@@ -76,9 +148,9 @@ chunks+="]"
 
 {
     echo "build_all=$build_all"
-    echo "build_libraries=$BUILD_LIBRARIES"
-    echo "build_static_sketches=$BUILD_STATIC_SKETCHES"
-    echo "build_idf=$BUILD_IDF"
+    echo "build_libraries=$LIB_CHANGED"
+    echo "build_static_sketches=$STATIC_SKETCHES_CHANGED"
+    echo "build_idf=$IDF_CHANGED"
     echo "chunk_count=$chunks_count"
     echo "chunks=$chunks"
 } >> "$GITHUB_OUTPUT"

.github/scripts/sketch_utils.sh

@@ -244,7 +244,7 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
         build_dir="$ARDUINO_BUILD_DIR"
     elif [ "$len" -eq 1 ]; then
         # build_dir="$sketchdir/build"
-        build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
+        build_dir="$HOME/.arduino/tests/$target/$sketchname/build.tmp"
     fi
 
     output_file="$HOME/.arduino/cli_compile_output.txt"
@@ -254,7 +254,7 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
     for i in $(seq 0 $((len - 1))); do
         if [ "$len" -ne 1 ]; then
             # build_dir="$sketchdir/build$i"
-            build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
+            build_dir="$HOME/.arduino/tests/$target/$sketchname/build$i.tmp"
         fi
         rm -rf "$build_dir"
         mkdir -p "$build_dir"

.github/scripts/tests_run.sh

@@ -11,9 +11,11 @@ function run_test {
     local error=0
     local sdkconfig_path
     local extra_args
+    local test_type
 
     sketchdir=$(dirname "$sketch")
     sketchname=$(basename "$sketchdir")
+    test_type=$(basename "$(dirname "$sketchdir")")
 
     if [ "$options" -eq 0 ] && [ -f "$sketchdir"/ci.json ]; then
         len=$(jq -r --arg target "$target" '.fqbn[$target] | length' "$sketchdir"/ci.json)
@@ -25,9 +27,9 @@ function run_test {
     fi
 
     if [ "$len" -eq 1 ]; then
-        sdkconfig_path="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig"
+        sdkconfig_path="$HOME/.arduino/tests/$target/$sketchname/build.tmp/sdkconfig"
     else
-        sdkconfig_path="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig"
+        sdkconfig_path="$HOME/.arduino/tests/$target/$sketchname/build0.tmp/sdkconfig"
     fi
 
     if [ -f "$sketchdir"/ci.json ]; then
@@ -43,22 +45,22 @@ function run_test {
     fi
 
     if [ ! -f "$sdkconfig_path" ]; then
-        printf "\033[93mSketch %s not built\nMight be due to missing target requirements or build failure\033[0m\n" "$sketchname"
+        printf "\033[93mSketch %s build not found in %s\nMight be due to missing target requirements or build failure\033[0m\n" "$(dirname "$sdkconfig_path")" "$sketchname"
         printf "\n\n\n"
         return 0
     fi
 
-    local right_target
-    right_target=$(grep -E "^CONFIG_IDF_TARGET=\"$target\"$" "$sdkconfig_path")
-    if [ -z "$right_target" ]; then
-        printf "\033[91mError: Sketch %s compiled for different target\n\033[0m\n" "$sketchname"
+    local compiled_target
+    compiled_target=$(grep -E "CONFIG_IDF_TARGET=" "$sdkconfig_path" | cut -d'"' -f2)
+    if [ "$compiled_target" != "$target" ]; then
+        printf "\033[91mError: Sketch %s compiled for %s, expected %s\033[0m\n" "$sketchname" "$compiled_target" "$target"
         printf "\n\n\n"
         return 1
     fi
 
     if [ "$len" -eq 1 ]; then
         # build_dir="$sketchdir/build"
-        build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
+        build_dir="$HOME/.arduino/tests/$target/$sketchname/build.tmp"
         report_file="$sketchdir/$target/$sketchname.xml"
     fi
 
@@ -81,7 +83,7 @@ function run_test {
 
         if [ "$len" -ne 1 ]; then
             # build_dir="$sketchdir/build$i"
-            build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
+            build_dir="$HOME/.arduino/tests/$target/$sketchname/build$i.tmp"
             report_file="$sketchdir/$target/$sketchname$i.xml"
         fi
 
@@ -113,14 +115,14 @@ function run_test {
         rm "$sketchdir"/diagram.json 2>/dev/null || true
 
         result=0
-        printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}"
-        bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$?
+        printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
+        bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
         printf "\n"
         if [ $result -ne 0 ]; then
             result=0
             printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i"
-            printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}"
-            bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$?
+            printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
+            bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
             printf "\n"
             if [ $result -ne 0 ]; then
                 printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i"

.github/workflows/build_py_tools.yml

@@ -12,7 +12,7 @@ on:
 jobs:
   find-changed-tools:
     name: Check if tools have been changed
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
       any_changed: ${{ steps.verify-changed-files.outputs.any_changed }}
       all_changed_files: ${{ steps.verify-changed-files.outputs.all_changed_files }}
@@ -30,16 +30,17 @@ jobs:
           echo "Make sure you are using a branch inside the repository and not a fork."
 
       - name: Verify Python Tools Changed
-        uses: tj-actions/changed-files@v41
         id: verify-changed-files
-        with:
-          fetch_depth: "2"
-          since_last_remote_commit: "true"
-          files: |
-            tools/get.py
-            tools/espota.py
-            tools/gen_esp32part.py
-            tools/gen_insights_package.py
+        run: |
+          CHANGED_FILES=$(git log -1 --name-only --pretty=format: -- tools/get.py tools/espota.py tools/gen_esp32part.py tools/gen_insights_package.py)
+          CHANGED_FILES=$(echo "$CHANGED_FILES" | tr '\n' ' ')
+          echo "all_changed_files=$CHANGED_FILES" >> $GITHUB_OUTPUT
+          if [ -n "$CHANGED_FILES" ]; then
+            echo "any_changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "any_changed=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: List all changed files
         shell: bash
         run: |
@@ -55,7 +56,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, macos-latest, ubuntu-20.04, ARM]
+        os: [windows-latest, macos-latest, ubuntu-latest, ubuntu-24.04-arm]
         include:
           - os: windows-latest
             TARGET: win64
@@ -64,14 +65,12 @@ jobs:
           - os: macos-latest
             TARGET: macos
             SEPARATOR: ":"
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
             TARGET: linux-amd64
             SEPARATOR: ":"
-          - os: ARM
-            CONTAINER: python:3.8-bullseye
+          - os: ubuntu-24.04-arm
             TARGET: arm
             SEPARATOR: ":"
-    container: ${{ matrix.CONTAINER }} # use python container on ARM
     env:
       DISTPATH: pytools-${{ matrix.TARGET }}
       PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi"
@@ -96,8 +95,6 @@ jobs:
           token: ${{ secrets.TOOLS_UPLOAD_PAT }}
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Set up Python 3.8
-        # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108
-        if: matrix.os != 'ARM'
         uses: actions/setup-python@master
         with:
           python-version: 3.8

.github/workflows/pre-commit.yml

@@ -58,7 +58,19 @@ jobs:
 
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/[email protected]
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUM: ${{ github.event.pull_request.number }}
+          IS_PR: ${{ github.event_name == 'pull_request' }}
+          GITHUB_SHA: ${{ github.sha }}
+        run: |
+          if [[ $IS_PR != 'true' ]]; then
+              files_changed=$(gh api repos/espressif/arduino-esp32/commits/"$GITHUB_SHA" --jq '.files[].filename')
+          else
+              files_changed=$(gh pr diff "$PR_NUM" --name-only)
+          fi
+          echo "all_changed_files=$files_changed" >> $GITHUB_OUTPUT
+          echo "Changed files: $files_changed"
 
       - name: Run pre-commit hooks in changed files
         run: pre-commit run --color=always --show-diff-on-failure --files ${{ steps.changed-files.outputs.all_changed_files }}

.github/workflows/push.yml

@@ -45,12 +45,13 @@ on:
       - "!.github/scripts/tests_*"
       - "!.github/scripts/upload_*"
       - "variants/esp32/**/*"
-      - "variants/esp32s2/**/*"
-      - "variants/esp32s3/**/*"
       - "variants/esp32c2/**/*"
       - "variants/esp32c3/**/*"
       - "variants/esp32c6/**/*"
       - "variants/esp32h2/**/*"
+      - "variants/esp32p4/**/*"
+      - "variants/esp32s2/**/*"
+      - "variants/esp32s3/**/*"
 
 concurrency:
   group: build-${{github.event.pull_request.number || github.ref}}
@@ -85,58 +86,13 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v44
-        with:
-          files_yaml: |
-            core:
-              - '.github/**'
-              - 'cores/**'
-              - 'package/**'
-              - 'tools/**'
-              - 'platform.txt'
-              - 'programmers.txt'
-              - "variants/esp32/**/*"
-              - "variants/esp32s2/**/*"
-              - "variants/esp32s3/**/*"
-              - "variants/esp32c3/**/*"
-              - "variants/esp32c6/**/*"
-              - "variants/esp32h2/**/*"
-            libraries:
-              - 'libraries/**/examples/**'
-              - 'libraries/**/src/**'
-            networking:
-              - 'libraries/Network/src/**'
-            fs:
-              - 'libraries/FS/src/**'
-            static_sketeches:
-              - 'libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino'
-              - 'libraries/BLE/examples/Server/Server.ino'
-              - 'libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino'
-              - 'libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino'
-              - 'libraries/NetworkClientSecure/src/**'
-              - 'libraries/BLE/src/**'
-              - 'libraries/Insights/src/**'
-            idf:
-              - 'idf_component.yml'
-              - 'Kconfig.projbuild'
-              - 'CMakeLists.txt'
-              - "variants/esp32c2/**/*"
-
       - name: Set chunks
         id: set-chunks
         env:
-          LIB_FILES: ${{ steps.changed-files.outputs.libraries_all_changed_files }}
           IS_PR: ${{ github.event_name == 'pull_request' }}
+          PR_NUM: ${{ github.event.pull_request.number }}
           MAX_CHUNKS: ${{ env.MAX_CHUNKS }}
-          BUILD_IDF: ${{ steps.changed-files.outputs.idf_any_changed == 'true' }}
-          BUILD_LIBRARIES: ${{ steps.changed-files.outputs.libraries_any_changed == 'true' }}
-          BUILD_STATIC_SKETCHES: ${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }}
-          FS_CHANGED: ${{ steps.changed-files.outputs.fs_any_changed == 'true' }}
-          NETWORKING_CHANGED: ${{ steps.changed-files.outputs.networking_any_changed == 'true' }}
-          CORE_CHANGED: ${{ steps.changed-files.outputs.core_any_changed == 'true' }}
-          LIB_CHANGED: ${{ steps.changed-files.outputs.libraries_any_changed == 'true' }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           bash ./.github/scripts/set_push_chunks.sh
 
@@ -237,15 +193,15 @@ jobs:
       needs.gen-chunks.outputs.build_all == 'true' ||
       needs.gen-chunks.outputs.build_libraries == 'true' ||
       needs.gen-chunks.outputs.build_idf == 'true'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         # The version names here correspond to the versions of espressif/idf Docker image.
         # See https://hub.docker.com/r/espressif/idf/tags and
         # https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/tools/idf-docker-image.html
         # for details.
-        idf_ver: ["release-v5.3"]
+        idf_ver: ["release-v5.4"]
         idf_target:
           [
             "esp32",