Merge branch 'test_ci'

Author: lucasssvaz

.github/workflows/build_tests.yml

@@ -47,11 +47,29 @@ jobs:
 
           echo "enabled=$enabled" >> $GITHUB_OUTPUT
 
-      - name: Checkout Repository
-        uses: actions/checkout@v4
+      - name: Checkout base repository
         if: ${{ steps.check-build.outputs.enabled == 'true' }}
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+            /*
+            !.github
+
+      - name: Checkout Scripts
+        if: ${{ steps.check-build.outputs.enabled == 'true' }}
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: .github/*
+
+      - name: Cat scripts
+        if: ${{ steps.check-build.outputs.enabled == 'true' }}
+        run: |
+          cat .github/scripts/tests_build.sh
 
       - name: Get libs cache
         uses: actions/cache@v4

.github/workflows/hw.yml

@@ -49,18 +49,32 @@ jobs:
 
           echo "enabled=$enabled" >> $GITHUB_OUTPUT
 
-      - name: Checkout repository
+      - name: Checkout base repository
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+            /*
+            !.github
 
-      - uses: actions/setup-python@v5
-        if: steps.check-tests.outputs.enabled == 'true'
+      - name: Checkout Scripts
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
+        uses: actions/checkout@v4
         with:
-          cache-dependency-path: tests/requirements.txt
-          cache: 'pip'
-          python-version: '3.10.1'
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: .github/*
+
+      # setup-python currently only works on ubuntu images
+      # - uses: actions/setup-python@v5
+      #   if: steps.check-tests.outputs.enabled == 'true'
+      #   with:
+      #     cache-dependency-path: tests/requirements.txt
+      #     cache: 'pip'
+      #     python-version: '3.10.1'
 
       - name: Install dependencies
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}

.github/workflows/pre-commit.yml

@@ -28,6 +28,8 @@ jobs:
       - name: Set up Python 3
         uses: actions/setup-python@v5
         with:
+          cache-dependency-path: tools/pre-commit/requirements.txt
+          cache: 'pip'
           python-version: "3.x"
 
       - name: Get Python version hash
@@ -41,11 +43,10 @@ jobs:
         with:
           path: |
             ~/.cache/pre-commit
-            ~/.cache/pip
-          key: pre-commit|${{ env.PY_HASH }}|${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml') }}
+          key: pre-commit-${{ env.PY_HASH }}-${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml', 'tools/pre-commit/requirements.txt') }}
 
       - name: Install python dependencies
-        run: python -m pip install pre-commit docutils
+        run: python -m pip install -r tools/pre-commit/requirements.txt
 
       - name: Get changed files
         id: changed-files
@@ -61,7 +62,6 @@ jobs:
         with:
           path: |
             ~/.cache/pre-commit
-            ~/.cache/pip
           key: ${{ steps.restore-cache.outputs.cache-primary-key }}
 
       - name: Push changes using pre-commit-ci-lite

.github/workflows/push.yml

@@ -48,6 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       build_all: ${{ steps.set-chunks.outputs.build_all }}
+      build_libraries: ${{ steps.set-chunks.outputs.build_libraries }}
       build_static_sketches: ${{ steps.set-chunks.outputs.build_static_sketches }}
       build_idf: ${{ steps.set-chunks.outputs.build_idf }}
       build_platformio: ${{ steps.set-chunks.outputs.build_platformio }}
@@ -103,6 +104,7 @@ jobs:
 
         build_platformio=${{ steps.changed-files.outputs.platformio_any_changed == 'true' }}
         build_idf=${{ steps.changed-files.outputs.idf_any_changed == 'true' }}
+        build_libraries=${{ steps.changed-files.outputs.libraries_any_changed == 'true' }}
         build_static_sketches=${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }}
 
         core_changed=${{ steps.changed-files.outputs.core_any_changed == 'true' }}
@@ -164,6 +166,7 @@ jobs:
         chunks+="]"
 
         echo "build_all=$build_all" >> $GITHUB_OUTPUT
+        echo "build_libraries=$build_libraries" >> $GITHUB_OUTPUT
         echo "build_static_sketches=$build_static_sketches" >> $GITHUB_OUTPUT
         echo "build_idf=$build_idf" >> $GITHUB_OUTPUT
         echo "build_platformio=$build_platformio" >> $GITHUB_OUTPUT
@@ -274,7 +277,10 @@ jobs:
   build-esp-idf-component:
     name: Build with ESP-IDF ${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
     needs: gen-chunks
-    if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_idf == 'true' }}
+    if: |
+      needs.gen-chunks.outputs.build_all == 'true' ||
+      needs.gen-chunks.outputs.build_libraries == 'true' ||
+      needs.gen-chunks.outputs.build_idf == 'true'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false

.github/workflows/qemu.yml

@@ -45,11 +45,24 @@ jobs:
 
           echo "enabled=$enabled" >> $GITHUB_OUTPUT
 
-      - name: Checkout repository
+      - name: Checkout base repository
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         uses: actions/checkout@v4
-        if: steps.check-tests.outputs.enabled == 'true'
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+            /*
+            !.github
+
+      - name: Checkout Scripts
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: .github/*
 
       - uses: actions/setup-python@v5
         if: steps.check-tests.outputs.enabled == 'true'

.github/workflows/tests.yml

@@ -67,7 +67,7 @@ jobs:
 
   call-build-tests:
     name: Build
-    uses: espressif/arduino-esp32/.github/workflows/build_tests.yml@master
+    uses: ./.github/workflows/build_tests.yml
     needs: gen-matrix
     if: github.event.action != 'closed'
     strategy:
@@ -80,7 +80,7 @@ jobs:
 
   call-hardware-tests:
     name: Hardware
-    uses: espressif/arduino-esp32/.github/workflows/hw.yml@master
+    uses: ./.github/workflows/hw.yml
     needs: [gen-matrix, call-build-tests]
     if: |
       github.repository == 'espressif/arduino-esp32' &&
@@ -97,7 +97,7 @@ jobs:
 
   call-wokwi-tests:
     name: Wokwi
-    uses: espressif/arduino-esp32/.github/workflows/wokwi.yml@master
+    uses: ./.github/workflows/wokwi.yml
     needs: [gen-matrix, call-build-tests]
     if: github.event.action != 'closed'
     strategy:
@@ -114,7 +114,7 @@ jobs:
   # This job is disabled for now
   call-qemu-tests:
     name: QEMU
-    uses: espressif/arduino-esp32/.github/workflows/qemu.yml@master
+    uses: ./.github/workflows/qemu.yml
     needs: [gen-matrix, call-build-tests]
     if: false
     strategy:
@@ -167,7 +167,7 @@ jobs:
   clean:
     name: Clean objects
     needs: unit-test-results
-    if: always() && ${{ github.event_name }} == 'pull_request_target' && ${{ github.event.action }} != 'closed'
+    if: always()
     permissions:
       actions: write
     runs-on: ubuntu-latest
@@ -179,6 +179,11 @@ jobs:
             const ref = '${{ github.event.pull_request.number || github.ref }}';
             const key_prefix = 'tests-' + ref + '-';
 
+            if ('${{ github.event_name }}' == 'pull_request_target' && '${{ github.event.action }}' != 'closed') {
+              console.log('Skipping cache cleanup for open PR');
+              return;
+            }
+
             await github.paginate(github.rest.actions.getActionsCacheList, {
               owner: context.repo.owner,
               repo: context.repo.repo,

.github/workflows/wokwi.yml

@@ -30,16 +30,15 @@ jobs:
       id: ${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }}-${{ inputs.chip }}-${{ inputs.type }}
     runs-on: ubuntu-latest
     steps:
-      # Disabled as Wokwi infrastrucutre is not stable (so we can re-trigger the tests manually)
-      # - name: Check if already run
-      #   if: ${{ github.event.pull_request.number != null }}
-      #   id: get-cache-results
-      #   uses: actions/cache/restore@v4
-      #   with:
-      #     key: tests-${{ env.id }}-results-wokwi
-      #     path: |
-      #       tests/**/*.xml
-      #       tests/**/result_*.json
+      - name: Check if already run
+        if: ${{ github.event.pull_request.number != null }}
+        id: get-cache-results
+        uses: actions/cache/restore@v4
+        with:
+          key: tests-${{ env.id }}-results-wokwi
+          path: |
+            tests/**/*.xml
+            tests/**/result_*.json
 
       - name: Evaluate if tests should be run
         id: check-tests
@@ -54,11 +53,24 @@ jobs:
 
           echo "enabled=$enabled" >> $GITHUB_OUTPUT
 
-      - name: Checkout Repository
+      - name: Checkout base repository
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         uses: actions/checkout@v4
-        if: steps.check-tests.outputs.enabled == 'true'
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+            /*
+            !.github
+
+      - name: Checkout Scripts
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: .github/*
 
       - uses: actions/setup-python@v5
         if: steps.check-tests.outputs.enabled == 'true'
@@ -100,15 +112,14 @@ jobs:
         run: |
           bash .github/scripts/tests_run.sh -c -type ${{ inputs.type }} -t ${{inputs.chip}} -i 0 -m 1 -W ${{env.WOKWI_TIMEOUT}}
 
-      # Disabled as Wokwi infrastrucutre is not stable (so we can re-trigger the tests manually)
-      # - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as cache
-      #   uses: actions/cache/save@v4
-      #   if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }}
-      #   with:
-      #     key: tests-${{ env.id }}-results-wokwi
-      #     path: |
-      #       tests/**/*.xml
-      #       tests/**/result_*.json
+      - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as cache
+        uses: actions/cache/save@v4
+        if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }}
+        with:
+          key: tests-${{ env.id }}-results-wokwi
+          path: |
+            tests/**/*.xml
+            tests/**/result_*.json
 
       - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as artifacts
         uses: actions/upload-artifact@v4

tools/get.py

@@ -107,12 +107,14 @@ def verify_files(filename, destination, rename_to):
                 for i, zipped_file in enumerate(archive.namelist(), 1):
                     local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
                     if not os.path.exists(local_path):
-                        print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
-                        print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+                        if verbose:
+                            print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
+                            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
                         return False
                     print_verification_progress(total_files, i, t1)
         except zipfile.BadZipFile:
-            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+            if verbose:
+                print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
             return False
     elif filename.endswith(".tar.gz"):
         try:
@@ -122,12 +124,14 @@ def verify_files(filename, destination, rename_to):
                 for i, zipped_file in enumerate(archive.getnames(), 1):
                     local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
                     if not os.path.exists(local_path):
-                        print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
-                        print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+                        if verbose:
+                            print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
+                            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
                         return False
                     print_verification_progress(total_files, i, t1)
         except tarfile.ReadError:
-            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+            if verbose:
+                print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
             return False
     elif filename.endswith(".tar.xz"):
         try:
@@ -137,12 +141,14 @@ def verify_files(filename, destination, rename_to):
                 for i, zipped_file in enumerate(archive.getnames(), 1):
                     local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
                     if not os.path.exists(local_path):
-                        print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
-                        print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+                        if verbose:
+                            print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
+                            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
                         return False
                     print_verification_progress(total_files, i, t1)
         except tarfile.ReadError:
-            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+            if verbose:
+                print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
             return False
     else:
         raise NotImplementedError("Unsupported archive type")
@@ -231,7 +237,12 @@ def unpack(filename, destination, force_extract):  # noqa: C901
             shutil.rmtree(rename_to)
         shutil.move(dirname, rename_to)
 
-    return True
+    if verify_files(filename, destination, rename_to):
+        print(" Files extracted successfully.")
+        return True
+    else:
+        print(" Failed to extract files.")
+        return False
 
 
 def download_file_with_progress(url, filename, start_time):
@@ -291,6 +302,7 @@ def get_tool(tool, force_download, force_extract):
     local_path = dist_dir + archive_name
     url = tool["url"]
     start_time = time.time()
+    print("")
     if not os.path.isfile(local_path) or force_download:
         if verbose:
             print("Downloading '" + archive_name + "' to '" + local_path + "'")
@@ -421,6 +433,9 @@ def identify_platform():
         current_dir + "/../package/package_esp32_index.template.json", identified_platform
     )
     mkdir_p(dist_dir)
+
+    print("\nDownloading and extracting tools...")
+
     for tool in tools_to_download:
         if is_test:
             print("Would install: {0}".format(tool["archiveFileName"]))
@@ -432,4 +447,4 @@ def identify_platform():
                     print(f"Tool {tool['archiveFileName']} was corrupted, but re-downloading did not help!\n")
                     sys.exit(1)
 
-    print("Platform Tools Installed")
+    print("\nPlatform Tools Installed")

tools/pre-commit/requirements.txt

@@ -0,0 +1,2 @@
+pre-commit==3.7.1
+docutils==0.21.2