test

lucasssvaz · lucasssvaz · commit 7e8f2fe1af5e · 2024-05-17T16:31:14.000+02:00
diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml
@@ -15,53 +15,64 @@ jobs:
       matrix:
         type: ['validation', 'performance']
         chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
+    env:
+      build-cache-key: tests-bin-${{ github.event.pull_request.number || github.ref }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha || github.sha }}
     steps:
       - name: Check if already built
         if: ${{ github.event.pull_request.number != null }}
         id: cache-build-binaries
         uses: actions/cache/restore@v4
         with:
           lookup-only: true
-          path: ~/.arduino/tests/*
-          key: tests-${{ github.event.pull_request.number }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha }}
+          key: ${{ env.build-cache-key }}
+          path: |
+            ~/.arduino/tests/**/build*.tmp/*.bin
+            ~/.arduino/tests/**/build*.tmp/*.elf
+            ~/.arduino/tests/**/build*.tmp/*.json
 
       - name: Evaluate if tests should be built
         id: check-build
         run: |
-          artifact_exists=${{ steps.cache-build-binaries.outputs.cache-hit == 'true' || false }}
+          cache_exists=${{ steps.cache-build-binaries.outputs.cache-hit == 'true' || false }}
           is_pr=${{ github.event.pull_request.number != null }}
           is_performance_test=${{ matrix.type == 'performance' }}
           is_performance_enabled=${{ contains(github.event.pull_request.labels.*.name, 'perf_test') }}
+          enabled=true
 
-          if [[ $artifact_exists == 'true' ]]; then
+          if [[ $cache_exists == 'true' ]]; then
             echo "Already built, skipping"
-            exit 0
+            enabled=false
           elif [[ $is_pr != 'true' ]]; then
             echo "Not a PR, building"
           elif [[ $is_performance_test == 'true' ]]; then
             if [[ $is_performance_enabled == 'true' ]]; then
               echo "Performance test enabled, building"
             else
               echo "Performance test disabled, skipping"
-              exit 0
+              enabled=false
             fi
           else
             echo "Validation test, building"
           fi
 
+          echo "enabled=$enabled" >> $GITHUB_OUTPUT
+
       - name: Checkout Repository
         uses: actions/checkout@v4
+        if: ${{ steps.check-build.outputs.enabled == 'true' }}
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Build sketches
+        if: ${{ steps.check-build.outputs.enabled == 'true' }}
         run: |
           bash .github/scripts/tests_build.sh -c -type ${{ matrix.type }} -t ${{ matrix.chip }}
 
       - name: Upload ${{ matrix.chip }}-${{ matrix.type }} binaries as cache
         uses: actions/cache/save@v4
+        if: ${{ steps.check-build.outputs.enabled == 'true' }}
         with:
-          key: tests-${{ github.event.pull_request.number || github.ref }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha || github.sha }}
+          key: ${{ env.build-cache-key }}
           path: |
             ~/.arduino/tests/**/build*.tmp/*.bin
             ~/.arduino/tests/**/build*.tmp/*.elf
diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml
@@ -1,7 +1,8 @@
 name: Clean objects
 
 on:
-  workflow_call:
+  pull_request_target:
+    types: [closed]
 
 jobs:
   clean:
diff --git a/.github/workflows/hw.yml b/.github/workflows/hw.yml
@@ -12,64 +12,95 @@ jobs:
     name: Hardware ${{ matrix.chip }} ${{ matrix.type }} tests
     runs-on: [arduino, "${{ matrix.chip }}"]
     strategy:
+      fail-fast: false
       matrix:
         type: ['validation', 'performance']
         chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
+    env:
+      cache-binaries-key: tests-bin-${{ github.event.pull_request.number || github.ref }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha || github.sha }}
+      cache-results-key: tests-hw-${{ github.event.pull_request.number || github.ref }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha || github.sha }}
     container:
       image: python:3.10.1-bullseye
       options: --privileged
     steps:
+      - name: Check if already built
+        if: ${{ github.event.pull_request.number != null }}
+        id: cache-results
+        uses: actions/cache/restore@v4
+        with:
+          lookup-only: true
+          key: ${{ env.cache-results-key }}
+          path: |
+            tests/**/*.xml
+            tests/**/result_*.json
+
       - name: Evaluate if tests should be run
         id: check-tests
         run: |
+          cache_exists=${{ steps.cache-results.outputs.cache-hit == 'true' || false }}
           is_pr=${{ github.event.pull_request.number != null }}
           is_performance_test=${{ matrix.type == 'performance' }}
           is_performance_enabled=${{ contains(github.event.pull_request.labels.*.name, 'perf_test') }}
+          enabled=true
 
-          if [[ $is_pr != 'true' ]]; then
+          if [[ $cache_exists == 'true' ]]; then
+            echo "Already ran, skipping"
+            enabled=false
+          elif [[ $is_pr != 'true' ]]; then
             echo "Not a PR, running"
           elif [[ $is_performance_test == 'true' ]]; then
             if [[ $is_performance_enabled == 'true' ]]; then
               echo "Performance test enabled, running"
             else
               echo "Performance test disabled, skipping"
-              exit 0
+              enabled=false
             fi
           else
             echo "Validation test, running"
           fi
 
+          echo "enabled=$enabled" >> $GITHUB_OUTPUT
+
       - name: Get binaries
         id: cache-build-binaries
         uses: actions/cache/restore@v4
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         with:
           fail-on-cache-miss: true
-          path: ~/.arduino/tests/*
-          key: tests-${{ github.event.pull_request.number || github.ref }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha || github.sha }}
+          key: ${{ env.cache-binaries-key }}
+          path: |
+            ~/.arduino/tests/**/build*.tmp/*.bin
+            ~/.arduino/tests/**/build*.tmp/*.elf
+            ~/.arduino/tests/**/build*.tmp/*.json
 
       - uses: actions/setup-python@v5
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         with:
           cache: 'pip'
           python-version: '3.10'
 
       - name: Checkout repository
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Install dependencies
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         run: |
           pip install -U pip
           pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
 
       - name: Run Tests
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         run: |
           bash .github/scripts/tests_run.sh -type ${{ matrix.type }} -t ${{ matrix.chip }} -e
 
-      - name: Upload test result artifacts
-        uses: actions/upload-artifact@v4
-        if: ${{ always() }}
+      - name: Upload ${{ matrix.chip }}-${{ matrix.type }} results as cache
+        uses: actions/cache/save@v4
+        if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }}
         with:
-          name: hw_results-${{ matrix.chip }}-${{ matrix.type }}}
-          if-no-files-found: error
+          key: ${{ env.cache-results-key }}
           path: |
             tests/**/*.xml
             tests/**/result_*.json
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -22,7 +22,8 @@ jobs:
     name: Run tests on hardware
     uses: ./.github/workflows/hw.yml
     needs: call-build-tests
-    if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'hil_test') }}
+    if: false
+    #if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'hil_test') }}
 
   # call-wokwi-tests:
   #   uses: ./.github/workflows/wokwi.yml
@@ -45,7 +46,7 @@ jobs:
 
   call-clean:
     name: Clean objects
-    needs: [call-build-tests, call-hardware-tests]
+    needs: [call-hardware-tests]
     uses: ./.github/workflows/clean.yml
     if: github.event_name != 'pull_request' || github.event.action == 'closed'
     permissions:
diff --git a/.github/workflows/wokwi.yml b/.github/workflows/wokwi.yml
@@ -1,96 +1,50 @@
-name: Run tests in wokwi on PR
+name: Wokwi tests
 
 on:
-  workflow_run:
-    workflows: [Run tests]
-    types:
-      - completed
+  workflow_call:
 
-permissions:
-  statuses: write
+concurrency:
+  group: tests-wokwi-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 env:
-  MAX_CHUNKS: 15
   WOKWI_TIMEOUT: 600000 # Milliseconds
   WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
-  get_event_file:
-    name: Get event file
-    runs-on: ubuntu-latest
-    outputs:
-      ref: ${{ steps.get-ref.outputs.ref }}
-    steps:
-      - name: Download event file
-        uses: actions/download-artifact@v4
-        with:
-          run-id: ${{github.event.workflow_run.id}}
-          github-token: ${{env.GITHUB_TOKEN}}
-          name: event_file
-
-      - name: Get ref
-        id: get-ref
-        run: |
-          PR_NUMBER=$(jq -r '.number' event.json)
-          echo "PR_NUMBER = $PR_NUMBER"
-          echo "ref=$PR_NUMBER" >> $GITHUB_OUTPUT
-
-  gen_chunks:
-    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
-    name: Generate Chunks matrix
-    runs-on: ubuntu-latest
-    needs: get_event_file
-    outputs:
-      chunks: ${{ steps.gen-chunks.outputs.chunks }}
-    concurrency:
-      group: wokwi-${{ needs.get_event_file.outputs.ref || github.ref }}
-      cancel-in-progress: true
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.workflow_run.head_sha  }} # Check out the code of the PR to generate accurate chunks
-
-      - name: Generate Chunks matrix
-        id: gen-chunks
-        run: |
-          set +e
-          .github/scripts/sketch_utils.sh count tests
-          sketches=$?
-          if [[ $sketches -ge ${{env.MAX_CHUNKS}} ]]; then
-            $sketches=${{env.MAX_CHUNKS}}
-          fi
-          set -e
-          rm sketches.txt
-          CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $((sketches - 1))`)
-          echo "chunks=${CHUNKS}" >>$GITHUB_OUTPUT
-
   wokwi-test:
-    needs: [get_event_file, gen_chunks]
-    name: ${{matrix.chip}}-Wokwi_Test#${{matrix.chunks}}
-    concurrency:
-      group: wokwi-${{ needs.get_event_file.outputs.ref || github.ref }}-${{matrix.chip}}-${{matrix.chunks}}
-      cancel-in-progress: true
+    name: Wokwi ${{ matrix.chip }} ${{ matrix.type }} tests
     strategy:
       fail-fast: false
       matrix:
         chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
-        chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
+        type: ['validation']
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Check if already run
+        if: ${{ github.event.pull_request.number != null }}
+        id: cache-results
+        uses: actions/cache/restore@v4
         with:
-          ref: ${{ github.event.workflow_run.head_sha }} # Check out the code of the PR to get correct pytest files
+          lookup-only: true
+          key: wokwi_results-${{ github.event.pull_request.number }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha }}
+          path: tests/**/*.xml
 
-      - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
-        uses: actions/download-artifact@v4
+      - name: Get binaries
+        id: cache-build-binaries
+        uses: actions/cache/restore@v4
+        with:
+          fail-on-cache-miss: true
+          key: tests-${{ github.event.pull_request.number || github.ref }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha || github.sha }}
+          path: |
+            ~/.arduino/tests/**/build*.tmp/*.bin
+            ~/.arduino/tests/**/build*.tmp/*.elf
+            ~/.arduino/tests/**/build*.tmp/*.json
+
+      - uses: actions/setup-python@v5
         with:
-          name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
-          path: ~/
-          run-id: ${{github.event.workflow_run.id}}
-          github-token: ${{env.GITHUB_TOKEN}}
+          cache: 'pip'
+          python-version: '3.10'
 
       - name: Install Wokwi CLI
         run: curl -L https://wokwi.com/ci/install.sh | sh
@@ -99,58 +53,17 @@ jobs:
         run: |
           pip install -U pip
           pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
-          sudo apt update && sudo apt install -y -qq jq
 
       - name: Run Tests
         run: |
-          bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -w ${{env.WOKWI_TIMEOUT}}
+          bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i 0 -m 1 -W ${{env.WOKWI_TIMEOUT}}
 
-      - name: Check if tests were skipped
-        id: check-test-skipped
-        run: |
-          if [ $(find "tests" -name ".test_skipped") ]; then
-            echo "skipped=true" >> $GITHUB_OUTPUT
-          else
-            echo "skipped=false" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Upload test result artifacts
-        uses: actions/upload-artifact@v4
-        if: ${{ always() && steps.check-test-skipped.outputs.skipped == 'false' }}
+      - name: Upload ${{ matrix.chip }}-${{ matrix.type }} binaries as cache
+        uses: actions/cache/save@v4
+        if: ${{ steps.check-build.outputs.enabled == 'true' }}
         with:
-          name: wokwi_results-${{matrix.chip}}-${{matrix.chunks}}
-          path: tests/**/*.xml
-
-  report-result:
-    name: Report wokwi test result
-    runs-on: ubuntu-latest
-    needs: [get_event_file, wokwi-test]
-    concurrency:
-      group: wokwi-${{ needs.get_event_file.outputs.ref || github.ref }}
-      cancel-in-progress: true
-    if: always() && github.event.workflow_run.event == 'pull_request'
-    steps:
-      - name: Report result
-        uses: actions/github-script@v7
-        with:
-          debug: true
-          script: |
-            const owner = '${{ github.repository_owner }}';
-            const repo = '${{ github.repository }}'.split('/')[1];
-            const sha = '${{ github.event.workflow_run.head_sha }}';
-            const result = '${{ needs.wokwi-test.result }}' == 'success' ? 'success' : 'failure';
-            core.debug(`owner: ${owner}`);
-            core.debug(`repo: ${repo}`);
-            core.debug(`sha: ${sha}`);
-            core.debug(`result: ${result}`);
-            const { context: name, state } = (await github.rest.repos.createCommitStatus({
-              context: 'Wokwi tests',
-              description: 'Wokwi simulator tests',
-              owner: owner,
-              repo: repo,
-              sha: sha,
-              state: result,
-              target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
-            })).data;
-            core.info(`${name} is ${state}`);
-
+          key: tests-${{ github.event.pull_request.number || github.ref }}-${{ matrix.chip }}-${{ matrix.type }}-${{ github.event.pull_request.head.sha || github.sha }}
+          path: |
+            ~/.arduino/tests/**/build*.tmp/*.bin
+            ~/.arduino/tests/**/build*.tmp/*.elf
+            ~/.arduino/tests/**/build*.tmp/*.json
