From a786b0e823bd89c3271d3ad0553cbbbaf4fe96b8 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 11 Jun 2025 18:26:48 +0200 Subject: [PATCH 1/4] test: log deploy output on timed out deploys --- tests/netlify-deploy.ts | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/netlify-deploy.ts b/tests/netlify-deploy.ts index bddec44fd1..2c4364e0d9 100644 --- a/tests/netlify-deploy.ts +++ b/tests/netlify-deploy.ts @@ -36,6 +36,8 @@ export class NextDeployInstance extends NextInstance { private _buildId: string private _deployId: string private _shouldDeleteDeploy: boolean = false + private _isCurrentlyDeploying: boolean = false + private _deployOutput: string = '' public get buildId() { // get deployment ID via fetch since we can't access @@ -52,6 +54,8 @@ export class NextDeployInstance extends NextInstance { return } + this._isCurrentlyDeploying = true + const setupStartTime = Date.now() // create the test site @@ -131,7 +135,7 @@ export class NextDeployInstance extends NextInstance { : testName const deployAlias = 'vercel-next-e2e' - const deployRes = await execa( + const deployResPromise = execa( 'npx', ['netlify', 'deploy', '--build', '--message', deployTitle ?? '', '--alias', deployAlias], { @@ -140,6 +144,15 @@ export class NextDeployInstance extends NextInstance { }, ) + const handleOutput = (chunk) => { + this._deployOutput += chunk + } + + deployResPromise.stdout.on('data', handleOutput) + deployResPromise.stderr.on('data', handleOutput) + + const deployRes = await deployResPromise + if (deployRes.exitCode !== 0) { throw new Error( `Failed to deploy project (${deployRes.exitCode}) ${deployRes.stdout} ${deployRes.stderr} `, @@ -168,7 +181,7 @@ export class NextDeployInstance extends NextInstance { require('console').log(`Logs: ${buildLogsUrl}`) } } catch (err) { - console.error(err) + require('console').error(err) throw new Error(`Failed to parse deploy output: ${deployRes.stdout}`) } @@ -181,6 +194,8 @@ export class NextDeployInstance extends NextInstance { require('console').log(`Got buildId: ${this._buildId}`) require('console').log(`Setup time: ${(Date.now() - setupStartTime) / 1000.0}s`) + + this._isCurrentlyDeploying = false } public async destroy(): Promise { @@ -205,6 +220,13 @@ export class NextDeployInstance extends NextInstance { } } + if (this._isCurrentlyDeploying) { + require('console').log('Destroying before deployment finished.') + if (this._deployOutput) { + require('console').log(`Deploy logs so far:\n\n${this._deployOutput}\n\n`) + } + } + await super.destroy() } From 7a22d9884631ba0e4bf93f075901d63efa4fd8bc Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 11 Jun 2025 18:30:33 +0200 Subject: [PATCH 2/4] tmp: test logging timed out deploys --- .github/workflows/run-tests.yml | 281 -------------------------------- .github/workflows/test-e2e.yml | 16 +- tests/netlify-deploy.ts | 5 + 3 files changed, 13 insertions(+), 289 deletions(-) delete mode 100644 .github/workflows/run-tests.yml diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml deleted file mode 100644 index c6e906ce2f..0000000000 --- a/.github/workflows/run-tests.yml +++ /dev/null @@ -1,281 +0,0 @@ -name: 'Run tests' -on: - pull_request: - branches: [main] - schedule: - - cron: '0 6 * * *' # Run every day at 6am UTC - workflow_dispatch: - inputs: - versions: - description: 'The versions of Next.js to test against (quoted and comma separated)' - required: false - default: 'latest' - -jobs: - setup: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Check PR labels - if: github.event_name == 'pull_request' - id: check-labels - uses: actions/github-script@v7 - with: - script: | - const { data: labels } = await github.rest.issues.listLabelsOnIssue({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - }); - return labels.some(label => label.name === 'autorelease: pending' || label.name === 'test all versions'); - - name: Set Next.js versions to test - id: set-matrix - # If this is the nightly build or a release PR then run the full matrix of versions - run: | - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "matrix=${{ github.event.inputs.versions }}" >> $GITHUB_OUTPUT - elif [ "${{ github.event_name }}" = "schedule" ] || [ "${{ steps.check-labels.outputs.result }}" = "true" ]; then - echo "matrix=[\"latest\", \"canary\", \"14.2.15\", \"13.5.1\"]" >> $GITHUB_OUTPUT - else - echo "matrix=[\"latest\"]" >> $GITHUB_OUTPUT - fi - - e2e: - needs: setup - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup.outputs.matrix) }} - shard: [1, 2, 3, 4, 5] - - steps: - - uses: actions/checkout@v4 - - name: 'Install Node' - uses: actions/setup-node@v4 - with: - node-version: '18.x' - cache: 'npm' - cache-dependency-path: '**/package-lock.json' - - uses: oven-sh/setup-bun@v2 - - name: setup pnpm/yarn - run: | - npm install -g corepack - corepack enable - shell: bash - - name: Install Deno - uses: denoland/setup-deno@v1 - with: - # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/build/blob/main/packages/edge-bundler/node/bridge.ts#L20 - deno-version: v1.46.3 - - name: 'Install dependencies' - run: npm ci - - name: 'Prepare Netlify CLI' - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - run: | - # Control netlify-cli as a regular dev dep but expose it globally for test fixtures to use - npm install -g "netlify-cli@$(npm list --json --depth=0 netlify-cli | jq -r ".dependencies[\"netlify-cli\"].version")" - npx netlify login - - name: Get installed Playwright version - id: playwright-version - run: echo "version=$(npm view @playwright/test version)" >> $GITHUB_OUTPUT - - uses: actions/cache@v4 - id: playwright-cache - with: - path: '~/.cache/ms-playwright' - key: '${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}' - restore-keys: | - ${{ runner.os }}-playwright- - - name: Install Playwright Browsers - if: steps.playwright-cache.outputs.cache-hit != 'true' - run: npx playwright install --with-deps - - name: Resolve Next.js version - id: resolve-next-version - shell: bash - run: | - RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) - echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT - echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" - - name: Run Playwright tests - run: npm run test:ci:e2e -- --shard=${{ matrix.shard }}/5 - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - NEXT_VERSION: ${{ matrix.version }} - NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} - - name: Upload blob report to GitHub Actions Artifacts - uses: actions/upload-artifact@v4 - if: always() - with: - name: blob-report-${{matrix.version}}-${{ matrix.shard }} - path: blob-report - retention-days: 1 - - test: - needs: setup - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] - os: [ubuntu-latest, windows-latest] - version: ${{ fromJson(needs.setup.outputs.matrix) }} - exclude: - - os: windows-latest - version: '13.5.1' - - os: windows-latest - version: '14.2.15' - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: 'Install Node' - uses: actions/setup-node@v4 - with: - node-version: '18.x' - cache: 'npm' - cache-dependency-path: '**/package-lock.json' - - name: Prefer npm global on windows - if: runner.os == 'Windows' - # On Windows by default PATH prefers corepack bundled with Node.js - # This prepends npm global to PATH to ensure that npm installed global corepack is used instead - run: | - echo "$(npm config get prefix)" >> "$GITHUB_PATH" - shell: bash - - name: setup pnpm/yarn - run: | - # global corepack installation requires --force on Windows, otherwise EEXIST errors occur - npm install -g corepack --force - corepack enable - shell: bash - - name: Install Deno - uses: denoland/setup-deno@v1 - with: - # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/e55f825bd985d3c92e21d1b765d71e70d5628fba/node/bridge.ts#L17 - deno-version: v1.46.3 - - name: 'Install dependencies' - run: npm ci - - name: 'Build' - run: npm run build - - name: 'Vendor deno helpers for integration tests' - run: node tools/vendor-deno-tools.js - - name: Resolve Next.js version - id: resolve-next-version - shell: bash - run: | - RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) - echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT - echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" - - name: Compute Fixtures Cache Key - id: fixture-cache-key - # Fixtures only need to be rebuilt if either fixture or support files change, - # so we're using a hash of the fixtures and support files as the cache key. - run: - echo "key=$(git ls-files -s tests/fixtures/ tests/utils/ tests/prepare.mjs | git hash-object --stdin)" - >> "$GITHUB_OUTPUT" - shell: bash - - name: Cache Fixtures - id: cache-fixtures - uses: actions/cache@v4 - with: - path: tests/fixtures - key: - integration-fixtures-${{ runner.os }}-${{steps.resolve-next-version.outputs.version}}-${{ - steps.fixture-cache-key.outputs.key }} - - name: 'Prepare Fixtures' - if: steps.cache-fixtures.outputs.cache-hit != 'true' - run: npm run pretest - env: - NEXT_VERSION: ${{ matrix.version }} - NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} - - name: 'Unit and integration tests' - run: npm run test:ci:unit-and-integration -- --shard=${{ matrix.shard }}/8 - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - NEXT_VERSION: ${{ matrix.version }} - NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} - TEMP: ${{ github.workspace }}/.. - - smoke: - if: always() - needs: setup - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - - name: 'Install Node' - uses: actions/setup-node@v4 - with: - node-version: '18.x' - cache: 'npm' - cache-dependency-path: '**/package-lock.json' - - name: setup pnpm/yarn - run: corepack enable - shell: bash - - name: Install Deno - uses: denoland/setup-deno@v1 - with: - # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/build/blob/main/packages/edge-bundler/node/bridge.ts#L20 - deno-version: v1.46.3 - - name: 'Install dependencies' - run: npm ci - - name: 'Build' - run: npm run build - - name: 'Prepare Netlify CLI' - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - run: | - # Control netlify-cli as a regular dev dep but expose it globally for test fixtures to use - npm install -g "netlify-cli@$(npm list --json --depth=0 netlify-cli | jq -r ".dependencies[\"netlify-cli\"].version")" - npx netlify login - - name: Resolve Next.js version - id: resolve-next-version - shell: bash - run: | - RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) - echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT - echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" - - name: 'Smoke tests' - run: npm run test:ci:smoke - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - NEXT_VERSION: ${{ matrix.version }} - NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} - - merge-reports: - if: always() - needs: [setup,e2e] - strategy: - fail-fast: false - matrix: - version: ${{ fromJson(needs.setup.outputs.matrix) }} - - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 18 - - name: Install dependencies - run: npm ci - - - name: Download blob reports from GitHub Actions Artifacts - uses: actions/download-artifact@v4 - with: - path: all-blob-reports - pattern: blob-report-${{ matrix.version }}-* - merge-multiple: true - - - name: Merge reports - run: | - npx playwright merge-reports --reporter html ./all-blob-reports - npx playwright merge-reports --reporter json ./all-blob-reports > merged_reports.json - - - name: Upload HTML report - uses: actions/upload-artifact@v4 - with: - name: html-report-${{ matrix.version }}-attempt-${{ github.run_attempt }} - path: playwright-report - retention-days: 14 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 593f1dcf65..005f40ae46 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -28,7 +28,7 @@ env: DATADOG_TRACE_NEXTJS_TEST: true DATADOG_API_KEY: foo TEST_CONCURRENCY: 2 - NEXT_E2E_TEST_TIMEOUT: 300000 + NEXT_E2E_TEST_TIMEOUT: 120000 NEXT_TELEMETRY_DISABLED: 1 NEXT_SKIP_NATIVE_POSTINSTALL: 1 TURBO_API: ${{ secrets.TURBO_API }} @@ -56,16 +56,16 @@ jobs: run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then VERSION_SELECTORS=[${{ github.event.inputs.versions }}] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT - echo "total=8" >> $GITHUB_OUTPUT + echo "group=[1]" >> $GITHUB_OUTPUT + echo "total=1" >> $GITHUB_OUTPUT elif [ "${{ github.event_name }}" == "pull_request" ]; then VERSION_SELECTORS=[\"latest\"] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT - echo "total=8" >> $GITHUB_OUTPUT + echo "group=[1]" >> $GITHUB_OUTPUT + echo "total=1" >> $GITHUB_OUTPUT else VERSION_SELECTORS=[\"latest\",\"canary\"] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT - echo "total=8" >> $GITHUB_OUTPUT + echo "group=[1]" >> $GITHUB_OUTPUT + echo "total=1" >> $GITHUB_OUTPUT fi VERSION_SPEC="[" @@ -198,7 +198,7 @@ jobs: NODE_ENV: production NEXT_EXTERNAL_TESTS_FILTERS: ${{ steps.test-filters.outputs.filters }} NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ steps.test-filters.outputs.skip-retry }} - run: node run-tests.js -g ${{ matrix.group }}/${{ needs.setup.outputs.total }} -c ${TEST_CONCURRENCY} --type e2e + run: node run-tests.js -g ${{ matrix.group }}/${{ needs.setup.outputs.total }} -c ${TEST_CONCURRENCY} --type e2e --test-pattern test/e2e/app-dir/app/index.test.ts working-directory: ${{ env.next-path }} - name: Upload Test Results diff --git a/tests/netlify-deploy.ts b/tests/netlify-deploy.ts index 2c4364e0d9..c415c1ceb2 100644 --- a/tests/netlify-deploy.ts +++ b/tests/netlify-deploy.ts @@ -153,6 +153,11 @@ export class NextDeployInstance extends NextInstance { const deployRes = await deployResPromise + // tmp - force timeout + await new Promise((resolve) => { + setTimeout(resolve, 120_000) + }) + if (deployRes.exitCode !== 0) { throw new Error( `Failed to deploy project (${deployRes.exitCode}) ${deployRes.stdout} ${deployRes.stderr} `, From e35aacc6a12da90d1ecfa0276d2b437e02223fe2 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 11 Jun 2025 20:31:44 +0200 Subject: [PATCH 3/4] Revert "tmp: test logging timed out deploys" This reverts commit 7a22d9884631ba0e4bf93f075901d63efa4fd8bc. --- .github/workflows/run-tests.yml | 281 ++++++++++++++++++++++++++++++++ .github/workflows/test-e2e.yml | 16 +- tests/netlify-deploy.ts | 5 - 3 files changed, 289 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/run-tests.yml diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000000..c6e906ce2f --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,281 @@ +name: 'Run tests' +on: + pull_request: + branches: [main] + schedule: + - cron: '0 6 * * *' # Run every day at 6am UTC + workflow_dispatch: + inputs: + versions: + description: 'The versions of Next.js to test against (quoted and comma separated)' + required: false + default: 'latest' + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Check PR labels + if: github.event_name == 'pull_request' + id: check-labels + uses: actions/github-script@v7 + with: + script: | + const { data: labels } = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + }); + return labels.some(label => label.name === 'autorelease: pending' || label.name === 'test all versions'); + - name: Set Next.js versions to test + id: set-matrix + # If this is the nightly build or a release PR then run the full matrix of versions + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "matrix=${{ github.event.inputs.versions }}" >> $GITHUB_OUTPUT + elif [ "${{ github.event_name }}" = "schedule" ] || [ "${{ steps.check-labels.outputs.result }}" = "true" ]; then + echo "matrix=[\"latest\", \"canary\", \"14.2.15\", \"13.5.1\"]" >> $GITHUB_OUTPUT + else + echo "matrix=[\"latest\"]" >> $GITHUB_OUTPUT + fi + + e2e: + needs: setup + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ${{ fromJson(needs.setup.outputs.matrix) }} + shard: [1, 2, 3, 4, 5] + + steps: + - uses: actions/checkout@v4 + - name: 'Install Node' + uses: actions/setup-node@v4 + with: + node-version: '18.x' + cache: 'npm' + cache-dependency-path: '**/package-lock.json' + - uses: oven-sh/setup-bun@v2 + - name: setup pnpm/yarn + run: | + npm install -g corepack + corepack enable + shell: bash + - name: Install Deno + uses: denoland/setup-deno@v1 + with: + # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/build/blob/main/packages/edge-bundler/node/bridge.ts#L20 + deno-version: v1.46.3 + - name: 'Install dependencies' + run: npm ci + - name: 'Prepare Netlify CLI' + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + run: | + # Control netlify-cli as a regular dev dep but expose it globally for test fixtures to use + npm install -g "netlify-cli@$(npm list --json --depth=0 netlify-cli | jq -r ".dependencies[\"netlify-cli\"].version")" + npx netlify login + - name: Get installed Playwright version + id: playwright-version + run: echo "version=$(npm view @playwright/test version)" >> $GITHUB_OUTPUT + - uses: actions/cache@v4 + id: playwright-cache + with: + path: '~/.cache/ms-playwright' + key: '${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}' + restore-keys: | + ${{ runner.os }}-playwright- + - name: Install Playwright Browsers + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: npx playwright install --with-deps + - name: Resolve Next.js version + id: resolve-next-version + shell: bash + run: | + RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) + echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT + echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" + - name: Run Playwright tests + run: npm run test:ci:e2e -- --shard=${{ matrix.shard }}/5 + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NEXT_VERSION: ${{ matrix.version }} + NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} + - name: Upload blob report to GitHub Actions Artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: blob-report-${{matrix.version}}-${{ matrix.shard }} + path: blob-report + retention-days: 1 + + test: + needs: setup + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + os: [ubuntu-latest, windows-latest] + version: ${{ fromJson(needs.setup.outputs.matrix) }} + exclude: + - os: windows-latest + version: '13.5.1' + - os: windows-latest + version: '14.2.15' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: 'Install Node' + uses: actions/setup-node@v4 + with: + node-version: '18.x' + cache: 'npm' + cache-dependency-path: '**/package-lock.json' + - name: Prefer npm global on windows + if: runner.os == 'Windows' + # On Windows by default PATH prefers corepack bundled with Node.js + # This prepends npm global to PATH to ensure that npm installed global corepack is used instead + run: | + echo "$(npm config get prefix)" >> "$GITHUB_PATH" + shell: bash + - name: setup pnpm/yarn + run: | + # global corepack installation requires --force on Windows, otherwise EEXIST errors occur + npm install -g corepack --force + corepack enable + shell: bash + - name: Install Deno + uses: denoland/setup-deno@v1 + with: + # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/e55f825bd985d3c92e21d1b765d71e70d5628fba/node/bridge.ts#L17 + deno-version: v1.46.3 + - name: 'Install dependencies' + run: npm ci + - name: 'Build' + run: npm run build + - name: 'Vendor deno helpers for integration tests' + run: node tools/vendor-deno-tools.js + - name: Resolve Next.js version + id: resolve-next-version + shell: bash + run: | + RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) + echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT + echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" + - name: Compute Fixtures Cache Key + id: fixture-cache-key + # Fixtures only need to be rebuilt if either fixture or support files change, + # so we're using a hash of the fixtures and support files as the cache key. + run: + echo "key=$(git ls-files -s tests/fixtures/ tests/utils/ tests/prepare.mjs | git hash-object --stdin)" + >> "$GITHUB_OUTPUT" + shell: bash + - name: Cache Fixtures + id: cache-fixtures + uses: actions/cache@v4 + with: + path: tests/fixtures + key: + integration-fixtures-${{ runner.os }}-${{steps.resolve-next-version.outputs.version}}-${{ + steps.fixture-cache-key.outputs.key }} + - name: 'Prepare Fixtures' + if: steps.cache-fixtures.outputs.cache-hit != 'true' + run: npm run pretest + env: + NEXT_VERSION: ${{ matrix.version }} + NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} + - name: 'Unit and integration tests' + run: npm run test:ci:unit-and-integration -- --shard=${{ matrix.shard }}/8 + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NEXT_VERSION: ${{ matrix.version }} + NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} + TEMP: ${{ github.workspace }}/.. + + smoke: + if: always() + needs: setup + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ${{ fromJson(needs.setup.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - name: 'Install Node' + uses: actions/setup-node@v4 + with: + node-version: '18.x' + cache: 'npm' + cache-dependency-path: '**/package-lock.json' + - name: setup pnpm/yarn + run: corepack enable + shell: bash + - name: Install Deno + uses: denoland/setup-deno@v1 + with: + # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/build/blob/main/packages/edge-bundler/node/bridge.ts#L20 + deno-version: v1.46.3 + - name: 'Install dependencies' + run: npm ci + - name: 'Build' + run: npm run build + - name: 'Prepare Netlify CLI' + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + run: | + # Control netlify-cli as a regular dev dep but expose it globally for test fixtures to use + npm install -g "netlify-cli@$(npm list --json --depth=0 netlify-cli | jq -r ".dependencies[\"netlify-cli\"].version")" + npx netlify login + - name: Resolve Next.js version + id: resolve-next-version + shell: bash + run: | + RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version) + echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT + echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'" + - name: 'Smoke tests' + run: npm run test:ci:smoke + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NEXT_VERSION: ${{ matrix.version }} + NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }} + + merge-reports: + if: always() + needs: [setup,e2e] + strategy: + fail-fast: false + matrix: + version: ${{ fromJson(needs.setup.outputs.matrix) }} + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Install dependencies + run: npm ci + + - name: Download blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@v4 + with: + path: all-blob-reports + pattern: blob-report-${{ matrix.version }}-* + merge-multiple: true + + - name: Merge reports + run: | + npx playwright merge-reports --reporter html ./all-blob-reports + npx playwright merge-reports --reporter json ./all-blob-reports > merged_reports.json + + - name: Upload HTML report + uses: actions/upload-artifact@v4 + with: + name: html-report-${{ matrix.version }}-attempt-${{ github.run_attempt }} + path: playwright-report + retention-days: 14 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 005f40ae46..593f1dcf65 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -28,7 +28,7 @@ env: DATADOG_TRACE_NEXTJS_TEST: true DATADOG_API_KEY: foo TEST_CONCURRENCY: 2 - NEXT_E2E_TEST_TIMEOUT: 120000 + NEXT_E2E_TEST_TIMEOUT: 300000 NEXT_TELEMETRY_DISABLED: 1 NEXT_SKIP_NATIVE_POSTINSTALL: 1 TURBO_API: ${{ secrets.TURBO_API }} @@ -56,16 +56,16 @@ jobs: run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then VERSION_SELECTORS=[${{ github.event.inputs.versions }}] - echo "group=[1]" >> $GITHUB_OUTPUT - echo "total=1" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT + echo "total=8" >> $GITHUB_OUTPUT elif [ "${{ github.event_name }}" == "pull_request" ]; then VERSION_SELECTORS=[\"latest\"] - echo "group=[1]" >> $GITHUB_OUTPUT - echo "total=1" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT + echo "total=8" >> $GITHUB_OUTPUT else VERSION_SELECTORS=[\"latest\",\"canary\"] - echo "group=[1]" >> $GITHUB_OUTPUT - echo "total=1" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT + echo "total=8" >> $GITHUB_OUTPUT fi VERSION_SPEC="[" @@ -198,7 +198,7 @@ jobs: NODE_ENV: production NEXT_EXTERNAL_TESTS_FILTERS: ${{ steps.test-filters.outputs.filters }} NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ steps.test-filters.outputs.skip-retry }} - run: node run-tests.js -g ${{ matrix.group }}/${{ needs.setup.outputs.total }} -c ${TEST_CONCURRENCY} --type e2e --test-pattern test/e2e/app-dir/app/index.test.ts + run: node run-tests.js -g ${{ matrix.group }}/${{ needs.setup.outputs.total }} -c ${TEST_CONCURRENCY} --type e2e working-directory: ${{ env.next-path }} - name: Upload Test Results diff --git a/tests/netlify-deploy.ts b/tests/netlify-deploy.ts index c415c1ceb2..2c4364e0d9 100644 --- a/tests/netlify-deploy.ts +++ b/tests/netlify-deploy.ts @@ -153,11 +153,6 @@ export class NextDeployInstance extends NextInstance { const deployRes = await deployResPromise - // tmp - force timeout - await new Promise((resolve) => { - setTimeout(resolve, 120_000) - }) - if (deployRes.exitCode !== 0) { throw new Error( `Failed to deploy project (${deployRes.exitCode}) ${deployRes.stdout} ${deployRes.stderr} `, From 30cc0f2868e032d0bfbe7458ce3a59d1a9161528 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 11 Jun 2025 20:34:13 +0200 Subject: [PATCH 4/4] test: clear deploy logs to avoid printing them twice --- tests/netlify-deploy.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/netlify-deploy.ts b/tests/netlify-deploy.ts index 2c4364e0d9..f5cc7ced43 100644 --- a/tests/netlify-deploy.ts +++ b/tests/netlify-deploy.ts @@ -154,6 +154,8 @@ export class NextDeployInstance extends NextInstance { const deployRes = await deployResPromise if (deployRes.exitCode !== 0) { + // clear deploy output to avoid printing it again in destroy() + this._deployOutput = '' throw new Error( `Failed to deploy project (${deployRes.exitCode}) ${deployRes.stdout} ${deployRes.stderr} `, )