Spawn a code-server instance for each test suite

This uses the current dev build by default but can be overidden with
CODE_SERVER_TEST_ENTRY (for example to test a release or some other
version).

Each instance has a separate state directory. This should make
parallelization work.

This also means you are no longer required to specify the password and
address yourself. `yarn test:e2e` should just work as-is.

Lastly, it means the tests are no longer subject to yarn watch randomly
restarting.

Author: code-asher

.github/workflows/ci.yaml

@@ -334,8 +334,9 @@ jobs:
     needs: package-linux-amd64
     runs-on: ubuntu-latest
     env:
-      PASSWORD: e45432jklfdsab
-      CODE_SERVER_ADDRESS: http://localhost:8080
+      # Since we build code-server we might as well run tests from the release
+      # since VS Code will load faster due to the bundling.
+      CODE_SERVER_TEST_ENTRY: "./release-packages/code-server*-linux-amd64/bin/code-server"
     steps:
       - uses: actions/checkout@v2
 
@@ -380,9 +381,7 @@ jobs:
           yarn install --check-files
 
       - name: Run end-to-end tests
-        run: |
-          ./release-packages/code-server*-linux-amd64/bin/code-server --log trace &
-          yarn test:e2e
+        run: yarn test:e2e
 
       - name: Upload test artifacts
         if: always()

ci/dev/test-e2e.sh

@@ -3,10 +3,30 @@ set -euo pipefail
 
 main() {
   cd "$(dirname "$0")/../.."
+
+  # Simple sanity checks to see that we've built. There could still be things
+  # wrong (native modules version issues, incomplete build, etc).
+  if [[ ! -d ./out ]]; then
+    echo >&2 "code-server has not been built"
+    echo >&2 "You can build it with 'yarn build' or 'yarn watch'"
+    exit 1
+  fi
+
+  if [[ ! -d ./lib/vscode/out ]]; then
+    echo >&2 "VS Code has not been built"
+    echo >&2 "You can build it with 'yarn build:vscode' or 'yarn watch'"
+    exit 1
+  fi
+
+  if [[ ! ${CODE_SERVER_TEST_ENTRY-} ]]; then
+    echo "Testing current dev build"
+    echo "Set CODE_SERVER_TEST_ENTRY to test another build of code-server"
+  else
+    echo "Testing build in '$CODE_SERVER_TEST_ENTRY'"
+  fi
+
   cd test
-  # We set these environment variables because they're used in the e2e tests
-  # they don't have to be these values, but these are the defaults
-  PASSWORD=e45432jklfdsab CODE_SERVER_ADDRESS=http://localhost:8080 yarn playwright test "$@"
+  yarn playwright test "$@"
 }
 
 main "$@"

test/e2e/baseFixture.ts

@@ -1,12 +1,47 @@
+import { field, logger } from "@coder/logger"
 import { test as base } from "@playwright/test"
-import { CodeServer } from "./models/CodeServer"
+import { CodeServer, CodeServerPage } from "./models/CodeServer"
 
-export const test = base.extend<{ codeServerPage: CodeServer }>({
-  codeServerPage: async ({ page }, use) => {
-    const codeServer = new CodeServer(page)
-    await codeServer.navigate()
-    await use(codeServer)
+/**
+ * Wraps `test.describe` to create and manage an instance of code-server. If you
+ * don't use this you will need to create your own code-server instance and pass
+ * it to `test.use`.
+ */
+export const describe = (name: string, fn: (codeServer: CodeServer) => void) => {
+  test.describe(name, () => {
+    // This will spawn on demand so nothing is necessary on before.
+    const codeServer = new CodeServer(name)
+
+    // Kill code-server after the suite has ended. This may happen even without
+    // doing it explicitly but it seems prudent to be sure.
+    test.afterAll(() => {
+      codeServer.close()
+    })
+
+    // This makes `codeServer` available to the extend call below.
+    test.use({ codeServer })
+
+    fn(codeServer)
+  })
+}
+
+interface TestFixtures {
+  codeServer: CodeServer
+  codeServerPage: CodeServerPage
+}
+
+/**
+ * Create a test that spawns code-server if necessary and ensures the page is
+ * ready.
+ */
+export const test = base.extend<TestFixtures>({
+  codeServer: undefined, // No default; should be provided through `test.use`.
+  codeServerPage: async ({ codeServer, page }, use) => {
+    const codeServerPage = new CodeServerPage(codeServer, page)
+    await codeServerPage.setup()
+    await use(codeServerPage)
   },
 })
 
+/** Shorthand for test.expect. */
 export const expect = test.expect

test/e2e/browser.test.ts

@@ -1,7 +1,7 @@
-import { test, expect } from "./baseFixture"
+import { describe, test, expect } from "./baseFixture"
 
 // This is a "gut-check" test to make sure playwright is working as expected
-test.describe("browser", () => {
+describe("browser", () => {
   test("browser should display correct userAgent", async ({ codeServerPage, browserName }) => {
     const displayNames = {
       chromium: "Chrome",

test/e2e/codeServer.test.ts

@@ -1,20 +1,20 @@
-import { CODE_SERVER_ADDRESS, storageState } from "../utils/constants"
-import { test, expect } from "./baseFixture"
+import { storageState } from "../utils/constants"
+import { describe, test, expect } from "./baseFixture"
 
-test.describe("CodeServer", () => {
+describe("CodeServer", () => {
   test.use({
     storageState,
   })
 
-  test(`should navigate to ${CODE_SERVER_ADDRESS}`, async ({ codeServerPage }) => {
+  test("should navigate to home page", async ({ codeServerPage }) => {
     // We navigate codeServer before each test
     // and we start the test with a storage state
     // which means we should be logged in
     // so it should be on the address
     const url = codeServerPage.page.url()
     // We use match because there may be a / at the end
     // so we don't want it to fail if we expect http://localhost:8080 to match http://localhost:8080/
-    expect(url).toMatch(CODE_SERVER_ADDRESS)
+    expect(url).toMatch(codeServerPage.address)
   })
 
   test("should always see the code-server editor", async ({ codeServerPage }) => {

test/e2e/globalSetup.test.ts

@@ -1,9 +1,9 @@
 import { storageState } from "../utils/constants"
-import { test, expect } from "./baseFixture"
+import { describe, test, expect } from "./baseFixture"
 
 // This test is to make sure the globalSetup works as expected
 // meaning globalSetup ran and stored the storageState
-test.describe("globalSetup", () => {
+describe("globalSetup", () => {
   test.use({
     storageState,
   })

test/e2e/login.test.ts

@@ -1,7 +1,7 @@
 import { PASSWORD } from "../utils/constants"
-import { test, expect } from "./baseFixture"
+import { describe, test, expect } from "./baseFixture"
 
-test.describe("login", () => {
+describe("login", () => {
   // Reset the browser so no cookies are persisted
   // by emptying the storageState
   test.use({

test/e2e/logout.test.ts

@@ -1,7 +1,7 @@
-import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
-import { test, expect } from "./baseFixture"
+import { PASSWORD } from "../utils/constants"
+import { describe, test, expect } from "./baseFixture"
 
-test.describe("logout", () => {
+describe("logout", () => {
   // Reset the browser so no cookies are persisted
   // by emptying the storageState
   test.use({
@@ -39,6 +39,6 @@ test.describe("logout", () => {
     // https://github.com/microsoft/playwright/issues/1987#issuecomment-620182151
     await Promise.all([codeServerPage.page.waitForNavigation(), codeServerPage.page.click(logoutButton)])
     const currentUrl = codeServerPage.page.url()
-    expect(currentUrl).toBe(`${CODE_SERVER_ADDRESS}/login`)
+    expect(currentUrl).toBe(`${codeServerPage.address}/login`)
   })
 })