ci: add e2e test verifying the deployed version of the runtime (#76)

* ci: add e2e test verifying the deployed version of the runtime

* chore: update secret name

* chore: update name

* chore: update auth details

* chore: use different token

* chore: update

* chore: fix cleanup

* chore: don't run e2e tests with vitest

* chore: update

Author: lukasholzer

.github/workflows/run-tests.yml

@@ -1,8 +1,51 @@
 name: 'Run tests'
 on:
   pull_request:
+    branches: [main]
 
 jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Install Node'
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+          cache: 'npm'
+          cache-dependency-path: '**/package-lock.json'
+      - name: 'Install dependencies'
+        run: npm ci
+      - name: 'Netlify Login'
+        env:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_TOKEN }}
+        run: |
+          npm i -g netlify-cli
+          netlify login
+      - name: Get installed Playwright version
+        id: playwright-version
+        run: echo "::set-output name=version::$(npm view @playwright/test version)"
+      - uses: actions/cache@v3
+        id: playwright-cache
+        with:
+          path: '~/.cache/ms-playwright'
+          key: '${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}'
+          restore-keys: |
+            ${{ runner.os }}-playwright-
+      - name: Install Playwright Browsers
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+        run: npx playwright install --with-deps
+      - name: Run Playwright tests
+        run: npm run e2e:ci
+        env:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_TOKEN }}
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
   test:
     runs-on: ubuntu-latest
     strategy:
@@ -16,7 +59,6 @@ jobs:
           node-version: '18.x'
           cache: 'npm'
           cache-dependency-path: '**/package-lock.json'
-
       - name: 'Install dependencies'
         run: npm ci
       - name: 'Build'

.gitignore

@@ -4,3 +4,7 @@ dist/
 
 # Local Netlify folder
 .netlify
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

package-lock.json

@@ -18,7 +18,9 @@
     "build:debug": "tsc --sourceMap",
     "build:watch": "tsc --watch",
     "test": "vitest",
-    "test:ci": "vitest run --reporter=default --retry=3"
+    "test:ci": "vitest run --reporter=default --retry=3",
+    "e2e": "playwright test",
+    "e2e:ci": "playwright test"
   },
   "repository": {
     "type": "git",
@@ -41,8 +43,8 @@
     "@netlify/build": "^29.20.6",
     "@netlify/functions": "^2.0.1",
     "@vercel/nft": "^0.24.3",
-    "fs-monkey": "^1.0.5",
     "fast-glob": "^3.3.2",
+    "fs-monkey": "^1.0.5",
     "os": "^0.1.2",
     "outdent": "^0.8.0",
     "p-limit": "^4.0.0"
@@ -51,6 +53,8 @@
     "@netlify/eslint-config-node": "^7.0.1",
     "@netlify/serverless-functions-api": "^1.10.1",
     "@netlify/zip-it-and-ship-it": "^9.25.5",
+    "@playwright/test": "^1.40.0",
+    "@types/node": "^20.9.2",
     "@types/uuid": "^9.0.6",
     "cheerio": "^1.0.0-rc.12",
     "execa": "^8.0.1",

package.json

@@ -0,0 +1,48 @@
+import { defineConfig, devices } from '@playwright/test'
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests/e2e',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: process.env.URL,
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  timeout: 10 * 60 * 1000,
+
+  /* Configure projects for major browsers */
+  projects: [
+    // {
+    //   name: 'setup',
+    //   testMatch: /global\.setup\.ts/,
+    // },
+    {
+      name: 'chromium',
+      // dependencies: ['setup'],
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+})

playwright.config.ts

@@ -0,0 +1,41 @@
+import { test, expect } from '@playwright/test'
+import { createE2EFixture } from '../utils/create-e2e-fixture.js'
+
+let ctx: Awaited<ReturnType<typeof createE2EFixture>>
+
+test.describe('page-router', () => {
+  test.beforeAll(async () => {
+    ctx = await createE2EFixture('page-router')
+  })
+
+  test.afterAll(async () => {
+    await ctx?.cleanup?.()
+  })
+
+  // NOT working yet as blob storage upload ins not working with the CLI
+  test.skip('Static revalidate works correctly', async ({ page }) => {
+    const response1 = await page.goto(new URL('static/revalidate', ctx.url).href)
+    const headers1 = response1?.headers() || {}
+    expect(response1?.status()).toBe(200)
+    expect(headers1['x-nextjs-cache']).toBe('HIT')
+    expect(headers1['netlify-cdn-cache-control']).toBe('s-maxage=3, stale-while-revalidate')
+
+    const h1 = await page.textContent('h1')
+    expect(h1).toBe('Show #71')
+
+    const date1 = await page.textContent('[data-testid="date-now"]')
+
+    // wait to have a stale page
+    page.waitForTimeout(3_000)
+
+    const response2 = await page.goto(new URL('static/revalidate', ctx.url).href)
+    const headers2 = response1?.headers() || {}
+    expect(response2?.status()).toBe(200)
+    expect(response1?.status()).toBe(200)
+    expect(headers2['x-nextjs-cache']).toBe('MISS')
+
+    const date2 = await page.textContent('[data-testid="date-now"]')
+
+    expect(date1).not.toBe(date2)
+  })
+})

tests/e2e/page-router.test.ts

@@ -0,0 +1,23 @@
+import { test, expect } from '@playwright/test'
+import { createE2EFixture } from '../utils/create-e2e-fixture.js'
+
+let ctx: Awaited<ReturnType<typeof createE2EFixture>>
+
+test.describe('simple-next-app', () => {
+  test.beforeAll(async () => {
+    ctx = await createE2EFixture('simple-next-app')
+  })
+
+  test.afterAll(async () => {
+    await ctx?.cleanup?.()
+  })
+
+  test('Renders the Home page correctly', async ({ page }) => {
+    await page.goto(ctx.url)
+
+    await expect(page).toHaveTitle('Simple Next App')
+
+    const h1 = await page.locator('h1')
+    await expect(h1).toHaveText('Home')
+  })
+})