From cdd3f82d00edc850f3f2517be0ca378a46554b47 Mon Sep 17 00:00:00 2001 From: per1234 Date: Mon, 12 Apr 2021 17:57:34 -0700 Subject: [PATCH] Set up basic integration tests Although the action's Python code itself is currently well tested via unit tests, this does not provide any checks for how the action interacts with the GitHub Actions framework and GitHub API. Even though it won't provide full verification of the report, simply running the action provides a "smoke test" as well as facilitating manual verification. On every pull request event, the action will run and source the sketches reports test data from a local path. When the job is triggered by an event from a fork, it will fail due to the access token not having the write permissions required to comment on the PR thread. The workflow is triggered to still pass under these conditions. Checking the log to see whether the action failed in the expected manner may still provide a useful validation of the PR. When the PR is from a branch of the repository, the deltas report will be commented on the PR thread. The workflow is also configured to run on push, schedule, workflow_dispatch, and repository_dispatch events. In this case it sources the sketches report from a workflow artifact. A dummy PR will be left permanently open in the repository with the sole purpose of serving as a target for these tests. --- .github/workflows/test-integration.yml | 54 ++++ .../arduino-avr-leonardo.json | 245 ++++++++++++++++++ .../arduino-sam-arduino_due_x.json | 245 ++++++++++++++++++ .../arduino-samd-mkrzero.json | 245 ++++++++++++++++++ README.md | 1 + 5 files changed, 790 insertions(+) create mode 100644 .github/workflows/test-integration.yml create mode 100644 .github/workflows/testdata/sketches-reports/arduino-avr-leonardo.json create mode 100644 .github/workflows/testdata/sketches-reports/arduino-sam-arduino_due_x.json create mode 100644 .github/workflows/testdata/sketches-reports/arduino-samd-mkrzero.json diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml new file mode 100644 index 0000000..502a666 --- /dev/null +++ b/.github/workflows/test-integration.yml @@ -0,0 +1,54 @@ +name: Run integration tests + +on: + pull_request: + paths: + - '.github/workflows/test-integration.yml' + - '.github/workflows/testdata/**' + - 'action.yml' + - 'Dockerfile' + - 'reportsizedeltas/**' + push: + paths: + - '.github/workflows/test-integration.yml' + - '.github/workflows/testdata/**' + - 'action.yml' + - 'Dockerfile' + - 'reportsizedeltas/**' + schedule: + # Run daily at 8 AM UTC to catch breakage caused by changes to external resources. + - cron: "0 8 * * *" + workflow_dispatch: + repository_dispatch: + +jobs: + local-source: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Run action + # Use arduino/report-size-deltas action from local path + uses: ./ + # The action will always fail on PRs submitted from forks due to not having write permissions. + # Some verification can still be achieved by checking the log to see whether it failed in the expected manner: + # WARNING:__main__:Temporarily unable to open URL (HTTP Error 403: Forbidden), retrying + # ... + # TimeoutError: Maximum number of URL load retries exceeded + continue-on-error: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + with: + sketches-reports-source: .github/workflows/testdata/sketches-reports + + artifact-source: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Run action + # Use arduino/report-size-deltas action from local path + uses: ./ diff --git a/.github/workflows/testdata/sketches-reports/arduino-avr-leonardo.json b/.github/workflows/testdata/sketches-reports/arduino-avr-leonardo.json new file mode 100644 index 0000000..64f9e7b --- /dev/null +++ b/.github/workflows/testdata/sketches-reports/arduino-avr-leonardo.json @@ -0,0 +1,245 @@ +{ + "commit_hash": "651f05f4d4aca30ac359e972c01568f873112d43", + "commit_url": "https://github.com/per1234/generate-size-deltas-report/commit/651f05f4d4aca30ac359e972c01568f873112d43", + "boards": [ + { + "board": "arduino:avr:leonardo", + "sketches": [ + { + "name": "examples/MIDIUSB_clock", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 28672, + "current": { + "absolute": 4792, + "relative": 16.71 + }, + "previous": { + "absolute": 4792, + "relative": 16.71 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 2560, + "current": { + "absolute": 443, + "relative": 17.3 + }, + "previous": { + "absolute": 443, + "relative": 17.3 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 2 + }, + "previous": { + "absolute": 2 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_loop", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 28672, + "current": { + "absolute": 4870, + "relative": 16.99 + }, + "previous": { + "absolute": 4870, + "relative": 16.99 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 2560, + "current": { + "absolute": 441, + "relative": 17.23 + }, + "previous": { + "absolute": 441, + "relative": 17.23 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 3 + }, + "previous": { + "absolute": 3 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_read", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 28672, + "current": { + "absolute": 4908, + "relative": 17.12 + }, + "previous": { + "absolute": 4908, + "relative": 17.12 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 2560, + "current": { + "absolute": 457, + "relative": 17.85 + }, + "previous": { + "absolute": 457, + "relative": 17.85 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 3 + }, + "previous": { + "absolute": 3 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_write", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 28672, + "current": { + "absolute": 4524, + "relative": 15.78 + }, + "previous": { + "absolute": 4564, + "relative": 15.92 + }, + "delta": { + "absolute": -40, + "relative": -0.14 + } + }, + { + "name": "RAM for global variables", + "maximum": 2560, + "current": { + "absolute": 197, + "relative": 7.7 + }, + "previous": { + "absolute": 213, + "relative": 8.32 + }, + "delta": { + "absolute": -16, + "relative": -0.62 + } + } + ], + "warnings": { + "current": { + "absolute": 3 + }, + "previous": { + "absolute": 3 + }, + "delta": { + "absolute": 0 + } + } + } + ], + "sizes": [ + { + "name": "flash", + "maximum": 28672, + "delta": { + "absolute": { + "minimum": -40, + "maximum": 0 + }, + "relative": { + "minimum": -0.14, + "maximum": 0.0 + } + } + }, + { + "name": "RAM for global variables", + "maximum": 2560, + "delta": { + "absolute": { + "minimum": -16, + "maximum": 0 + }, + "relative": { + "minimum": -0.62, + "maximum": 0.0 + } + } + } + ], + "warnings": { + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/.github/workflows/testdata/sketches-reports/arduino-sam-arduino_due_x.json b/.github/workflows/testdata/sketches-reports/arduino-sam-arduino_due_x.json new file mode 100644 index 0000000..89a0ace --- /dev/null +++ b/.github/workflows/testdata/sketches-reports/arduino-sam-arduino_due_x.json @@ -0,0 +1,245 @@ +{ + "commit_hash": "651f05f4d4aca30ac359e972c01568f873112d43", + "commit_url": "https://github.com/per1234/generate-size-deltas-report/commit/651f05f4d4aca30ac359e972c01568f873112d43", + "boards": [ + { + "board": "arduino:sam:arduino_due_x", + "sketches": [ + { + "name": "examples/MIDIUSB_clock", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 524288, + "current": { + "absolute": 11792, + "relative": 2.25 + }, + "previous": { + "absolute": 11792, + "relative": 2.25 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": "N/A", + "current": { + "absolute": "N/A", + "relative": "N/A" + }, + "previous": { + "absolute": "N/A", + "relative": "N/A" + }, + "delta": { + "absolute": "N/A", + "relative": "N/A" + } + } + ], + "warnings": { + "current": { + "absolute": 5 + }, + "previous": { + "absolute": 5 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_loop", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 524288, + "current": { + "absolute": 11640, + "relative": 2.22 + }, + "previous": { + "absolute": 11640, + "relative": 2.22 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": "N/A", + "current": { + "absolute": "N/A", + "relative": "N/A" + }, + "previous": { + "absolute": "N/A", + "relative": "N/A" + }, + "delta": { + "absolute": "N/A", + "relative": "N/A" + } + } + ], + "warnings": { + "current": { + "absolute": 6 + }, + "previous": { + "absolute": 6 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_read", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 524288, + "current": { + "absolute": 11808, + "relative": 2.25 + }, + "previous": { + "absolute": 11808, + "relative": 2.25 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": "N/A", + "current": { + "absolute": "N/A", + "relative": "N/A" + }, + "previous": { + "absolute": "N/A", + "relative": "N/A" + }, + "delta": { + "absolute": "N/A", + "relative": "N/A" + } + } + ], + "warnings": { + "current": { + "absolute": 6 + }, + "previous": { + "absolute": 6 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_write", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 524288, + "current": { + "absolute": 11680, + "relative": 2.23 + }, + "previous": { + "absolute": 11680, + "relative": 2.23 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": "N/A", + "current": { + "absolute": "N/A", + "relative": "N/A" + }, + "previous": { + "absolute": "N/A", + "relative": "N/A" + }, + "delta": { + "absolute": "N/A", + "relative": "N/A" + } + } + ], + "warnings": { + "current": { + "absolute": 6 + }, + "previous": { + "absolute": 6 + }, + "delta": { + "absolute": 0 + } + } + } + ], + "sizes": [ + { + "name": "flash", + "maximum": 524288, + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + }, + "relative": { + "minimum": 0.0, + "maximum": 0.0 + } + } + }, + { + "name": "RAM for global variables", + "maximum": "N/A", + "delta": { + "absolute": { + "minimum": "N/A", + "maximum": "N/A" + }, + "relative": { + "minimum": "N/A", + "maximum": "N/A" + } + } + } + ], + "warnings": { + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/.github/workflows/testdata/sketches-reports/arduino-samd-mkrzero.json b/.github/workflows/testdata/sketches-reports/arduino-samd-mkrzero.json new file mode 100644 index 0000000..fb56a1c --- /dev/null +++ b/.github/workflows/testdata/sketches-reports/arduino-samd-mkrzero.json @@ -0,0 +1,245 @@ +{ + "commit_hash": "651f05f4d4aca30ac359e972c01568f873112d43", + "commit_url": "https://github.com/per1234/generate-size-deltas-report/commit/651f05f4d4aca30ac359e972c01568f873112d43", + "boards": [ + { + "board": "arduino:samd:mkrzero", + "sketches": [ + { + "name": "examples/MIDIUSB_clock", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 262144, + "current": { + "absolute": 11976, + "relative": 4.57 + }, + "previous": { + "absolute": 11892, + "relative": 4.54 + }, + "delta": { + "absolute": 84, + "relative": 0.03 + } + }, + { + "name": "RAM for global variables", + "maximum": 32768, + "current": { + "absolute": 2300, + "relative": 7.02 + }, + "previous": { + "absolute": 2300, + "relative": 7.02 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 3 + }, + "previous": { + "absolute": 3 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_loop", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 262144, + "current": { + "absolute": 11784, + "relative": 4.5 + }, + "previous": { + "absolute": 11784, + "relative": 4.5 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 32768, + "current": { + "absolute": 2296, + "relative": 7.01 + }, + "previous": { + "absolute": 2296, + "relative": 7.01 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 4 + }, + "previous": { + "absolute": 4 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_read", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 262144, + "current": { + "absolute": 11940, + "relative": 4.55 + }, + "previous": { + "absolute": 11968, + "relative": 4.57 + }, + "delta": { + "absolute": -28, + "relative": -0.01 + } + }, + { + "name": "RAM for global variables", + "maximum": 32768, + "current": { + "absolute": 2296, + "relative": 7.01 + }, + "previous": { + "absolute": 2296, + "relative": 7.01 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + } + ], + "warnings": { + "current": { + "absolute": 4 + }, + "previous": { + "absolute": 4 + }, + "delta": { + "absolute": 0 + } + } + }, + { + "name": "examples/MIDIUSB_write", + "compilation_success": true, + "sizes": [ + { + "name": "flash", + "maximum": 262144, + "current": { + "absolute": 11808, + "relative": 4.5 + }, + "previous": { + "absolute": 11808, + "relative": 4.5 + }, + "delta": { + "absolute": 0, + "relative": 0.0 + } + }, + { + "name": "RAM for global variables", + "maximum": 32768, + "current": { + "absolute": 2064, + "relative": 6.3 + }, + "previous": { + "absolute": 2032, + "relative": 6.2 + }, + "delta": { + "absolute": 32, + "relative": 0.1 + } + } + ], + "warnings": { + "current": { + "absolute": 4 + }, + "previous": { + "absolute": 4 + }, + "delta": { + "absolute": 0 + } + } + } + ], + "sizes": [ + { + "name": "flash", + "maximum": 262144, + "delta": { + "absolute": { + "minimum": -28, + "maximum": 84 + }, + "relative": { + "minimum": -0.01, + "maximum": 0.03 + } + } + }, + { + "name": "RAM for global variables", + "maximum": 32768, + "delta": { + "absolute": { + "minimum": 0, + "maximum": 32 + }, + "relative": { + "minimum": 0.0, + "maximum": 0.1 + } + } + } + ], + "warnings": { + "delta": { + "absolute": { + "minimum": 0, + "maximum": 0 + } + } + } + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index c7e9d06..aa896fe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # `arduino/report-size-deltas` action [![Tests](https://github.com/arduino/report-size-deltas/workflows/libraries/report-size-deltas%20workflow/badge.svg)](https://github.com/arduino/report-size-deltas/actions?workflow=libraries/report-size-deltas+workflow) +[![Integration Tests](https://github.com/arduino/report-size-deltas/actions/workflows/test-integration.yml/badge.svg)](https://github.com/arduino/report-size-deltas/actions/workflows/test-integration.yml) [![Spell Check](https://github.com/arduino/report-size-deltas/workflows/Spell%20Check/badge.svg)](https://github.com/arduino/report-size-deltas/actions?workflow=Spell+Check) [![codecov](https://codecov.io/gh/arduino/report-size-deltas/branch/master/graph/badge.svg)](https://codecov.io/gh/arduino/report-size-deltas)