test-integration.yml

name: Run integration tests

on:
  pull_request:
    paths:
      - '.github/workflows/test-integration.yml'
      - '.github/workflows/testdata/**'
      - 'action.yml'
      - 'action-setup.sh'
      - 'compilesketches/**'

  push:
    paths:
      - '.github/workflows/test-integration.yml'
      - '.github/workflows/testdata/**'
      - 'action.yml'
      - 'action-setup.sh'
      - 'compilesketches/**'

env:
  SKETCHES_REPORTS_PATH: sketches-reports
  TESTDATA_SKETCHES_PATH: .github/workflows/testdata/sketches
  TESTDATA_REPORTS_PATH:  .github/workflows/testdata/reports

jobs:
  default-inputs:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Servo library
        uses: actions/checkout@v2
        with:
          repository: arduino-libraries/servo
          ref: 1.1.7

      - name: Checkout local repo
        uses: actions/checkout@v2
        with:
          # Must be checked out to a subfolder to not interfere with the checked out library under test
          path: extras/compile-sketches

      - name: Run action with default input values
        # Use action from local path
        uses: ./extras/compile-sketches


  all-inputs:
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false

      matrix:
        board:
          - fqbn: arduino:avr:uno
            platforms: |
              - name: arduino:avr
                version: 1.8.3
            libraries: |
              - name: Servo
                version: 1.1.7
          # Board that requires Boards Manager URL
          - fqbn: esp8266:esp8266:huzzah
            platforms: |
              - name: esp8266:esp8266
                source-url: https://arduino.esp8266.com/stable/package_esp8266com_index.json
                version: 2.7.4
            libraries: |
              # The official Servo library is not compatible with ESP8266, but that platform has a bundled Servo lib
              # so there are no library dependencies
              -

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Run action
        # Use action from local path
        uses: ./
        with:
          cli-version: 0.15.1
          platforms: ${{ matrix.board.platforms }}
          fqbn: ${{ matrix.board.fqbn }}
          libraries: ${{ matrix.board.libraries }}
          sketch-paths: |
            - ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum
            - ${{ env.TESTDATA_SKETCHES_PATH }}/ServoLibrary
            - ${{ env.TESTDATA_SKETCHES_PATH }}/TestCompileFlags
          cli-compile-flags: |
            - --build-property
            - compiler.cpp.extra_flags="-DSTRING_MACRO="hello world"" -DINT_MACRO=2 -DFLAG_MACRO
            - --export-binaries
          enable-deltas-report: true
          enable-warnings-report: true
          sketches-report-path: ${{ env.SKETCHES_REPORTS_PATH }}
          verbose: true

      - name: Verify --export-binaries flag was used by compilation command
        run: |
          [ -d ${{ env.TESTDATA_SKETCHES_PATH }}/BareMinimum/build ]

      - name: Set report artifact name
        id: report-artifact-name
        run: |
          echo "::set-output name=report-artifact-name::${{ github.job }}"

      - name: Save sketches report as workflow artifact
        uses: actions/upload-artifact@v2
        with:
          path: sketches-reports
          name: ${{ steps.report-artifact-name.outputs.report-artifact-name }}

    outputs:
      report-artifact-name: ${{ steps.report-artifact-name.outputs.report-artifact-name }}


  check-sketches-reports:
    needs: all-inputs
    runs-on: ubuntu-latest

    steps:
      # Checkout is needed to get the golden reports
      - name: Checkout local repo
        uses: actions/checkout@v2

      - name: Download sketches reports artifact
        uses: actions/download-artifact@v2
        with:
          name: ${{ needs.all-inputs.outputs.report-artifact-name }}
          path: ${{ env.SKETCHES_REPORTS_PATH }}

      - name: Compare generated sketches report to golden report
        run: |
          # The commit hash changes on every push, so it can't be compared against the golden report
          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
            COMMIT_HASH="${{ github.event.pull_request.head.sha }}"
          else
            COMMIT_HASH="$(git rev-parse HEAD)"
          fi

          set +o errexit  # Unset errexit so the results of all report comparisons are shown before exiting
          EXIT_STATUS=0
          while read -r reportPath; do
            echo "Sketches report $reportPath matches expected:"

            (
              # Check static report components against golden reports
              jq \
                --null-input \
                --exit-status \
                --slurpfile generated "$reportPath" \
                --slurpfile golden "${{ env.TESTDATA_REPORTS_PATH }}/${{ needs.all-inputs.outputs.report-artifact-name }}/$(basename "$reportPath")" \
                '($generated|.[0].boards) == ($golden|.[0].boards)'
            ) && (
              # Check the commit_hash value
              jq \
                --null-input \
                --exit-status \
                --slurpfile generated "$reportPath" \
                --arg commit_hash "$COMMIT_HASH" \
                '$generated|.[0].commit_hash == $commit_hash|.'
            ) && (
              # Check the commit_url value
              jq \
                --null-input \
                --exit-status \
                --slurpfile generated "$reportPath" \
                --arg commit_url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${COMMIT_HASH}" \
                '$generated|.[0].commit_url == $commit_url|.'
            )

            if [[ $? -ne 0 ]]; then
              echo "::error::Sketches report $reportPath mismatch."
              EXIT_STATUS=1
            fi
          done <<<"$(find "${{ env.SKETCHES_REPORTS_PATH }}" -type f -and -name '*.json')"

          exit $EXIT_STATUS


  expected-failed-compilation:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout local repo
        uses: actions/checkout@v2

      - name: Compile sketch that is expected to error
        id: compile-sketches
        continue-on-error: true
        uses: ./
        with:
          fqbn: arduino:avr:uno
          libraries: |
            -
          sketch-paths: |
            - ${{ env.TESTDATA_SKETCHES_PATH }}/Error

      - name: Fail the job if the action run succeeded
        if: steps.compile-sketches.outcome == 'success'
        run: |
          echo "::error::The action run was expected to fail, but passed!"
          exit 1