diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index 513fd16b371..bce7afe5668 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -11,9 +11,11 @@ function run_test { local error=0 local sdkconfig_path local extra_args + local test_type sketchdir=$(dirname "$sketch") sketchname=$(basename "$sketchdir") + test_type=$(basename "$(dirname "$sketchdir")") if [ "$options" -eq 0 ] && [ -f "$sketchdir"/ci.json ]; then len=$(jq -r --arg target "$target" '.fqbn[$target] | length' "$sketchdir"/ci.json) @@ -113,14 +115,14 @@ function run_test { rm "$sketchdir"/diagram.json 2>/dev/null || true result=0 - printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}" - bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$? + printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}" + bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$? printf "\n" if [ $result -ne 0 ]; then result=0 printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i" - printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}" - bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$? + printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}" + bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$? printf "\n" if [ $result -ne 0 ]; then printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i" diff --git a/.github/workflows/tests_results.yml b/.github/workflows/tests_results.yml index 947de64ece5..9e213fad14e 100644 --- a/.github/workflows/tests_results.yml +++ b/.github/workflows/tests_results.yml @@ -41,6 +41,7 @@ jobs: original_sha=$(cat ./artifacts/parent-artifacts/sha.txt) original_ref=$(cat ./artifacts/parent-artifacts/ref.txt) original_conclusion=$(cat ./artifacts/parent-artifacts/conclusion.txt) + original_run_id=$(cat ./artifacts/parent-artifacts/run_id.txt) # Sanitize the values to avoid security issues @@ -59,17 +60,27 @@ jobs: # Conclusion: Allow alphabetical characters and underscores original_conclusion=$(echo "$original_conclusion" | tr -cd '[:alpha:]_') + # Run ID: Allow numeric characters + original_run_id=$(echo "$original_run_id" | tr -cd '[:digit:]') + echo "original_event=$original_event" >> $GITHUB_ENV echo "original_action=$original_action" >> $GITHUB_ENV echo "original_sha=$original_sha" >> $GITHUB_ENV echo "original_ref=$original_ref" >> $GITHUB_ENV echo "original_conclusion=$original_conclusion" >> $GITHUB_ENV + echo "original_run_id=$original_run_id" >> $GITHUB_ENV echo "original_event = $original_event" echo "original_action = $original_action" echo "original_sha = $original_sha" echo "original_ref = $original_ref" echo "original_conclusion = $original_conclusion" + echo "original_run_id = $original_run_id" + + - name: Print links to other runs + run: | + echo "Build, Hardware and QEMU tests: https://github.com/${{ github.repository }}/actions/runs/${{ env.original_run_id }}" + echo "Wokwi tests: https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}" - name: Publish Unit Test Results uses: EnricoMi/publish-unit-test-result-action@v2 @@ -80,6 +91,17 @@ jobs: files: ./artifacts/**/*.xml action_fail: true compare_to_earlier_commit: false + json_file: ./unity_results.json + json_suite_details: true + + - name: Upload JSON + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: unity_results + overwrite: true + path: | + ./unity_results.json - name: Fail if tests failed if: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' }} @@ -138,11 +160,18 @@ jobs: })).data; core.info(`${name} is ${state}`); - - name: Create output folder + - name: Generate report if: ${{ !cancelled() && (env.original_event == 'schedule' || env.original_event == 'workflow_dispatch') }} # codespell:ignore cancelled + env: + REPORT_FILE: ./runtime-tests-results/RUNTIME_TESTS_REPORT.md + WOKWI_RUN_ID: ${{ github.event.workflow_run.id }} + BUILD_RUN_ID: ${{ env.original_run_id }} + IS_FAILING: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' || job.status == 'failure' }} run: | - rm -rf artifacts - mkdir -p runtime-tests-results + rm -rf artifacts $REPORT_FILE + mv -f ./unity_results.json ./runtime-tests-results/unity_results.json + touch $REPORT_FILE + python3 ./runtime-tests-results/table_generator.py ./runtime-tests-results/unity_results.json >> $REPORT_FILE - name: Generate badge if: ${{ !cancelled() && (env.original_event == 'schedule' || env.original_event == 'workflow_dispatch') }} # codespell:ignore cancelled @@ -161,6 +190,6 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" if [[ `git status --porcelain` ]]; then git add --all - git commit -m "Updated runtime tests badge" + git commit -m "Updated runtime tests report" git push origin HEAD:gh-pages fi diff --git a/.github/workflows/tests_wokwi.yml b/.github/workflows/tests_wokwi.yml index f5eb2efcad2..c254d5fa153 100644 --- a/.github/workflows/tests_wokwi.yml +++ b/.github/workflows/tests_wokwi.yml @@ -109,6 +109,10 @@ jobs: printf "\nAction = " cat artifacts/action.txt + printf "${{ github.event.workflow_run.id }}" >> artifacts/run_id.txt + printf "\nRun ID = " + cat artifacts/run_id.txt + if [ -z "$ref" ] || [ "$ref" == "null" ]; then echo "Failed to get PR number or ref" exit 1 diff --git a/README.md b/README.md index 0a9a6d0440c..9d300b25c54 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build Status](https://img.shields.io/github/actions/workflow/status/espressif/arduino-esp32/push.yml?branch=master&event=push&label=Compilation%20Tests)](https://github.com/espressif/arduino-esp32/actions/workflows/push.yml?query=branch%3Amaster+event%3Apush) [![Verbose Build Status](https://img.shields.io/github/actions/workflow/status/espressif/arduino-esp32/push.yml?branch=master&event=schedule&label=Compilation%20Tests%20(Verbose))](https://github.com/espressif/arduino-esp32/actions/workflows/push.yml?query=branch%3Amaster+event%3Aschedule) [![External Libraries Test](https://img.shields.io/github/actions/workflow/status/espressif/arduino-esp32/lib.yml?branch=master&event=schedule&label=External%20Libraries%20Test)](https://github.com/espressif/arduino-esp32/blob/gh-pages/LIBRARIES_TEST.md) -[![Runtime Tests](https://github.com/espressif/arduino-esp32/blob/gh-pages/runtime-tests-results/badge.svg)](https://github.com/espressif/arduino-esp32/actions/workflows/tests_results.yml) +[![Runtime Tests](https://github.com/espressif/arduino-esp32/blob/gh-pages/runtime-tests-results/badge.svg)](https://github.com/espressif/arduino-esp32/blob/gh-pages/runtime-tests-results/RUNTIME_TESTS_REPORT.md) ### Need help or have a question? Join the chat at [Discord](https://discord.gg/8xY6e9crwv) or [open a new Discussion](https://github.com/espressif/arduino-esp32/discussions)