ci: run tests against different versions (#314)

* ci: run tests against different versions

* ci: ignore lockfiles for pnpm in fixtures

* chore: allow manual trigger

* chore: install next version in e2e

* chore: log install command

* chore: use matrix for unit and integration tests

* chore: patch package.jsons

* chore: add fixture dep

* chore: handle errors in patching

* chore: allow fixtures and tests to exclude versions

* chore: update output syntax

* chore: check version in integration tests too

* chore: update actions

* chore: handle errors reading package.json

* chore: don't patch inside node_modules

* chore: add doc

* chore: allow setting site id

* chore: fix pnpm test

* chore: artifact name

* chore: make stream test conditional

* chore: format with prettier

* chore: fixes and logging for prepare script

* chore: shard e2e tests

* chore: merge reports

* chore: correct reporter

* chore: handle more version constraints

* test: hacky mocking for request-context leak test to work with dynamic import and require

* test: provide NEXT_VERSION env var also to vitest execution step so smoke tests update next versions as well

* test: don't rely on pnpm-lock.yaml to decide on package-manager

* test: handle packageManager field not being set

* test: drop commented out debug code

* revert pnpm-lock.yaml changes

* test: allow updating pnpm lock in vitest fixtures

* test: use --force on vitest tests with pnpm

* test: revert changing integration fixture pnpm command

* test: use actual resolved next version to calculate fixture cache key

* test: don't revert version, otherwise pnpm lock won't match and if you run against non default NEXT_VERSION some mess in working dir seems acceptable to make things work

* test: mock requests instead of spinning actual http server because 13.5.1 doesn't seem to allow overwriting env vars on execution, so we can't change fetch target dynamically

* don't install repo dependencies via pnpm, we use npm for that

* chore: format with prettier

* actually don't use custom pnpm action at all, we are updating dependencies so pnpm locks are not to be trusted for caching

* separate playwright reports for each next version

* fix blob report path ?

---------

Co-authored-by: ascorbic <[email protected]>
Co-authored-by: Michal Piechowiak <[email protected]>
Co-authored-by: pieh <[email protected]>

Author: ascorbic

.github/actions/pnpm/action.yaml

@@ -2,10 +2,53 @@ name: 'Run tests'
 on:
   pull_request:
     branches: [main]
+  schedule:
+    - cron: '0 6 * * *' # Run every day at 6am UTC
+  workflow_dispatch:
+    inputs:
+      versions:
+        description: 'The versions of Next.js to test against (quoted and comma separated)'
+        required: false
+        default: 'latest'
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Check PR labels
+        if: github.event_name == 'pull_request'
+        id: check-labels
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { data: labels } = await github.rest.issues.listLabelsOnIssue({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.payload.pull_request.number,
+            });
+            return labels.some(label => label.name === 'autorelease: pending' || label.name === 'test all versions');
+      - name: Set Next.js versions to test
+        id: set-matrix
+        # If this is the nightly build or a release PR then run the full matrix of versions
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "matrix=${{ github.event.inputs.versions }}" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event_name }}" = "schedule" ] || [ "${{ steps.check-labels.outputs.result }}" = "true" ]; then
+            echo "matrix=[\"latest\", \"canary\", \"13.5.1\"]" >> $GITHUB_OUTPUT
+          else
+            echo "matrix=[\"latest\"]" >> $GITHUB_OUTPUT
+          fi
   e2e:
+    needs: setup
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJson(needs.setup.outputs.matrix) }}
+        shard: [1, 2, 3, 4]
+
     steps:
       - uses: actions/checkout@v4
       - name: 'Install Node'
@@ -15,8 +58,9 @@ jobs:
           cache: 'npm'
           cache-dependency-path: '**/package-lock.json'
       - uses: oven-sh/setup-bun@v1
-      - name: Setup PNPM
-        uses: ./.github/actions/pnpm
+      - name: setup pnpm/yarn
+        run: corepack enable
+        shell: bash
       - name: Install Deno
         uses: denoland/setup-deno@v1
         with:
@@ -33,7 +77,7 @@ jobs:
       - name: Get installed Playwright version
         id: playwright-version
         run: echo "::set-output name=version::$(npm view @playwright/test version)"
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         id: playwright-cache
         with:
           path: '~/.cache/ms-playwright'
@@ -44,23 +88,27 @@ jobs:
         if: steps.playwright-cache.outputs.cache-hit != 'true'
         run: npx playwright install --with-deps
       - name: Run Playwright tests
-        run: npm run e2e:ci
+        run: npm run e2e:ci -- --shard=${{ matrix.shard }}/4
         env:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_TOKEN }}
-      - uses: actions/upload-artifact@v3
+          NEXT_VERSION: ${{ matrix.version }}
+      - name: Upload blob report to GitHub Actions Artifacts
+        uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+          name: blob-report-${{matrix.version}}-${{ matrix.shard }}
+          path: blob-report
+          retention-days: 1
 
   test:
+    needs: setup
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        shard: [1/8, 2/8, 3/8, 4/8, 5/8, 6/8, 7/8, 8/8]
+        shard: [1, 2, 3, 4, 5, 6, 7, 8]
         os: [ubuntu-latest, windows-latest]
+        version: ${{ fromJson(needs.setup.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v4
       - name: 'Install Node'
@@ -69,8 +117,9 @@ jobs:
           node-version: '18.x'
           cache: 'npm'
           cache-dependency-path: '**/package-lock.json'
-      - name: Setup PNPM
-        uses: ./.github/actions/pnpm
+      - name: setup pnpm/yarn
+        run: corepack enable
+        shell: bash
       - name: Install Deno
         uses: denoland/setup-deno@v1
         with:
@@ -86,6 +135,9 @@ jobs:
         run: |
           npm i -g netlify-cli
           netlify login
+      - name: Get actual Next.js version
+        id: actual-next-version
+        run: echo "version=$(npm view next@${{ matrix.version }} version)" >> $GITHUB_OUTPUT
       - name: Compute Fixtures Cache Key
         id: fixture-cache-key
         # Fixtures only need to be rebuilt if anything in the tests/fixtures directory changes,
@@ -99,12 +151,50 @@ jobs:
         uses: actions/cache@v4
         with:
           path: tests/fixtures
-          key: integration-fixtures-${{ runner.os }}-${{ steps.fixture-cache-key.outputs.key }}
+          key:
+            integration-fixtures-${{ runner.os }}-${{steps.next-version.outputs.version}}-${{
+            steps.fixture-cache-key.outputs.key }}
       - name: 'Prepare Fixtures'
         if: steps.cache-fixtures.outputs.cache-hit != 'true'
         run: npm run pretest
+        env:
+          NEXT_VERSION: ${{ matrix.version }}
       - name: 'Test'
-        run: npm run test:ci -- --shard=${{ matrix.shard }}
+        run: npm run test:ci -- --shard=${{ matrix.shard }}/8
         env:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_TOKEN }}
-          TEMP: ${{ github.workspace }}/..
+          NEXT_VERSION: ${{ matrix.version }}
+          TEMP: ${{ github.workspace }}/..
+  merge-reports:
+    if: always()
+    needs: [setup,e2e]
+    strategy:
+      fail-fast: false
+      matrix:
+        version: ${{ fromJson(needs.setup.outputs.matrix) }}
+
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Download blob reports from GitHub Actions Artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: all-blob-reports
+        pattern: blob-report-${{ matrix.version }}-*
+        merge-multiple: true
+
+    - name: Merge into HTML Report
+      run: npx playwright merge-reports --reporter html ./all-blob-reports 
+
+    - name: Upload HTML report
+      uses: actions/upload-artifact@v4
+      with:
+        name: html-report-${{ matrix.version }}-attempt-${{ github.run_attempt }}
+        path: playwright-report
+        retention-days: 14

.github/workflows/run-tests.yml

@@ -74,7 +74,7 @@ jobs:
         run: echo "PATH=$(pnpm store path --silent)" >> ${GITHUB_OUTPUT}
 
       - name: cache pnpm deps
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ steps.pnpm-store.outputs.PATH }}
           key: pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
@@ -86,7 +86,7 @@ jobs:
         run: echo "PATH=$(npm config get cache)" >> ${GITHUB_OUTPUT}
 
       - name: cache npm deps
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ steps.npm-cache.outputs.PATH }}
           key: node-cache-${{ hashFiles('**/package-lock.json') }}

.github/workflows/test-e2e.yml

@@ -14,3 +14,4 @@ deno.lock
 .eslintcache
 /report/index.html
 .DS_Store
+tests/fixtures/**/package-lock.json

.gitignore

@@ -10,5 +10,6 @@ deno.lock
 tests/fixtures/simple-next-app-dist-dir/cool/output
 .nx
 custom-dist-dir
+pnpm.lock
 # to avoid needing extra permissions to format files
 .github/workflows

.prettierignore

@@ -28,6 +28,16 @@ For a simple next.js app
 
 ## Testing
 
+The repo includes two types of tests: e2e tests in the repo that use Playwright, integration tests
+that use Vitest.
+
+By default the e2e and integration tests run against the latest version of Next.js. To run tests
+against a specific version, set the `NEXT_VERSION` environment variable to the desired version.
+
+By default, PRs will run the integration tests against the latest version of Next.js. To run tests
+against `latest`, `canary` and `13.5.1`, apply the `test all versions` label to the PR when you
+create it. These also run nightly and on release PRs.
+
 ### Integration testing
 
 How to add new integration test scenarios to the application:
@@ -75,6 +85,12 @@ following:
 > [!TIP] If you'd like to always keep the deployment and the local fixture around for
 > troubleshooting, run `E2E_PERSIST=1 npm run e2e`.
 
+### Next.js tests
+
+There is a script `run-local-test.sh` and GitHub workflow that runs the e2e tests from the Next.js
+repo against this repo. It requires that `next.js` is checked out in the same parent directory as
+this repo, and is run from this repo with `./run-local-test.sh your-test-pattern-here`.
+
 #### cleanup old deploys
 
 To cleanup old and dangling deploys from failed builds you can run the following script:

README.md

@@ -82,6 +82,7 @@
     "picomatch": "^3.0.1",
     "prettier": "^3.2.5",
     "regexp-tree": "^0.1.27",
+    "semver": "^7.6.0",
     "typescript": "^5.1.6",
     "unionfs": "^4.5.1",
     "uuid": "^9.0.1",

package-lock.json

@@ -15,7 +15,7 @@ export default defineConfig({
   workers: process.env.CI ? 3 : undefined,
   globalSetup: './tests/test-setup-e2e.ts',
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  reporter: process.env.CI ? 'blob' : 'list',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */