feat(testing): add e2e tests for code-server and terminal

src/node/constants.ts

@@ -1,5 +1,6 @@
 import { logger } from "@coder/logger"
 import { JSONSchemaForNPMPackageJsonFiles } from "@schemastore/package"
+import * as os from "os"
 import * as path from "path"
 
 export function getPackageJson(relativePath: string): JSONSchemaForNPMPackageJsonFiles {
@@ -18,3 +19,4 @@ const pkg = getPackageJson("../../package.json")
 export const version = pkg.version || "development"
 export const commit = pkg.commit || "development"
 export const rootPath = path.resolve(__dirname, "../..")
+export const tmpdir = path.join(os.tmpdir(), "code-server")

src/node/socket.ts

@@ -4,7 +4,8 @@ import * as path from "path"
 import * as tls from "tls"
 import { Emitter } from "../common/emitter"
 import { generateUuid } from "../common/util"
-import { canConnect, tmpdir } from "./util"
+import { tmpdir } from "./constants"
+import { canConnect } from "./util"
 
 /**
  * Provides a way to proxy a TLS socket. Can be used when you need to pass a

src/node/util.ts

@@ -8,8 +8,6 @@ import * as path from "path"
 import * as util from "util"
 import xdgBasedir from "xdg-basedir"
 
-export const tmpdir = path.join(os.tmpdir(), "code-server")
-
 interface Paths {
   data: string
   config: string

test/e2e/browser.test.ts

@@ -1,15 +1,23 @@
 import { test, expect } from "@playwright/test"
-import { CODE_SERVER_ADDRESS } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 // This is a "gut-check" test to make sure playwright is working as expected
-test("browser should display correct userAgent", async ({ page, browserName }) => {
-  const displayNames = {
-    chromium: "Chrome",
-    firefox: "Firefox",
-    webkit: "Safari",
-  }
-  await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
-  const userAgent = await page.evaluate("navigator.userAgent")
-
-  expect(userAgent).toContain(displayNames[browserName])
+test.describe("browser", () => {
+  let codeServer: CodeServer
+
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.navigate()
+  })
+
+  test("browser should display correct userAgent", async ({ page, browserName }) => {
+    const displayNames = {
+      chromium: "Chrome",
+      firefox: "Firefox",
+      webkit: "Safari",
+    }
+    const userAgent = await page.evaluate("navigator.userAgent")
+
+    expect(userAgent).toContain(displayNames[browserName])
+  })
 })

test/e2e/codeServer.test.ts

@@ -0,0 +1,45 @@
+import { test, expect } from "@playwright/test"
+import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
+
+test.describe("CodeServer", () => {
+  // Create a new context with the saved storage state
+  // so we don't have to logged in
+  const options: any = {}
+  let codeServer: CodeServer
+
+  // TODO@jsjoeio
+  // Fix this once https://github.com/microsoft/playwright-test/issues/240
+  // is fixed
+  if (STORAGE) {
+    const storageState = JSON.parse(STORAGE) || {}
+    options.contextOptions = {
+      storageState,
+    }
+  }
+
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.setup()
+  })
+
+  test(`should navigate to ${CODE_SERVER_ADDRESS}`, options, async ({ page }) => {
+    // We navigate codeServer before each test
+    // and we start the test with a storage state
+    // which means we should be logged in
+    // so it should be on the address
+    const url = page.url()
+    // We use match because there may be a / at the end
+    // so we don't want it to fail if we expect http://localhost:8080 to match http://localhost:8080/
+    expect(url).toMatch(CODE_SERVER_ADDRESS)
+  })
+
+  test("should always see the code-server editor", options, async ({ page }) => {
+    expect(await codeServer.isEditorVisible()).toBe(true)
+  })
+
+  test("should show the Integrated Terminal", options, async ({ page }) => {
+    await codeServer.focusTerminal()
+    expect(await page.isVisible("#terminal")).toBe(true)
+  })
+})

test/e2e/globalSetup.test.ts

@@ -1,12 +1,14 @@
 import { test, expect } from "@playwright/test"
-import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
+import { STORAGE } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 // This test is to make sure the globalSetup works as expected
 // meaning globalSetup ran and stored the storageState in STORAGE
 test.describe("globalSetup", () => {
   // Create a new context with the saved storage state
   // so we don't have to logged in
   const options: any = {}
+  let codeServer: CodeServer
 
   // TODO@jsjoeio
   // Fix this once https://github.com/microsoft/playwright-test/issues/240
@@ -17,9 +19,12 @@ test.describe("globalSetup", () => {
       storageState,
     }
   }
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.setup()
+  })
   test("should keep us logged in using the storageState", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Make sure the editor actually loaded
-    expect(await page.isVisible("div.monaco-workbench"))
+    expect(await codeServer.isEditorVisible()).toBe(true)
   })
 })

test/e2e/login.test.ts

@@ -1,5 +1,6 @@
 import { test, expect } from "@playwright/test"
-import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
+import { PASSWORD } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 test.describe("login", () => {
   // Reset the browser so no cookies are persisted
@@ -9,26 +10,32 @@ test.describe("login", () => {
       storageState: {},
     },
   }
+  let codeServer: CodeServer
+
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.navigate()
+  })
 
   test("should see the login page", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // It should send us to the login page
     expect(await page.title()).toBe("code-server login")
   })
 
   test("should be able to login", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
     await page.waitForLoadState("networkidle")
+    // We do this because occassionally code-server doesn't load on Firefox
+    // but loads if you reload once or twice
+    await codeServer.reloadUntilEditorIsVisible()
     // Make sure the editor actually loaded
-    expect(await page.isVisible("div.monaco-workbench"))
+    expect(await codeServer.isEditorVisible()).toBe(true)
   })
 
   test("should see an error message for missing password", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Skip entering password
     // Click the submit button and login
     await page.click(".submit")
@@ -37,7 +44,6 @@ test.describe("login", () => {
   })
 
   test("should see an error message for incorrect password", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", "password123")
     // Click the submit button and login
@@ -47,7 +53,6 @@ test.describe("login", () => {
   })
 
   test("should hit the rate limiter for too many unsuccessful logins", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", "password123")
     // Click the submit button and login

test/e2e/logout.test.ts

@@ -1,5 +1,6 @@
 import { test, expect } from "@playwright/test"
 import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 test.describe("logout", () => {
   // Reset the browser so no cookies are persisted
@@ -9,22 +10,31 @@ test.describe("logout", () => {
       storageState: {},
     },
   }
+  let codeServer: CodeServer
+
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.navigate()
+  })
+
   test("should be able login and logout", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
     await page.waitForLoadState("networkidle")
+    // We do this because occassionally code-server doesn't load on Firefox
+    // but loads if you reload once or twice
+    await codeServer.reloadUntilEditorIsVisible()
     // Make sure the editor actually loaded
-    expect(await page.isVisible("div.monaco-workbench"))
+    expect(await codeServer.isEditorVisible()).toBe(true)
 
     // Click the Application menu
     await page.click("[aria-label='Application Menu']")
 
     // See the Log out button
     const logoutButton = "a.action-menu-item span[aria-label='Log out']"
-    expect(await page.isVisible(logoutButton))
+    expect(await page.isVisible(logoutButton)).toBe(true)
 
     await page.hover(logoutButton)
     // TODO(@jsjoeio)

test/e2e/models/CodeServer.ts

@@ -0,0 +1,104 @@
+import { Page } from "playwright"
+import { CODE_SERVER_ADDRESS } from "../../utils/constants"
+// This is a Page Object Model
+// We use these to simplify e2e test authoring
+// See Playwright docs: https://playwright.dev/docs/pom/
+export class CodeServer {
+  page: Page
+
+  constructor(page: Page) {
+    this.page = page
+  }
+
+  /**
+   * Navigates to CODE_SERVER_ADDRESS
+   */
+  async navigate() {
+    await this.page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+  }
+
+  /**
+   * Checks if the editor is visible
+   * and reloads until it is
+   */
+  async reloadUntilEditorIsVisible() {
+    const editorIsVisible = await this.isEditorVisible()
+    let reloadCount = 0
+
+    // Occassionally code-server timeouts in Firefox
+    // we're not sure why
+    // but usually a reload or two fixes it
+    // TODO@jsjoeio @oxy look into Firefox reconnection/timeout issues
+    while (!editorIsVisible) {
+      reloadCount += 1
+      if (await this.isEditorVisible()) {
+        console.log(`    Editor became visible after ${reloadCount} reloads`)
+        break
+      }
+      // When a reload happens, we want to wait for all resources to be
+      // loaded completely. Hence why we use that instead of DOMContentLoaded
+      // Read more: https://thisthat.dev/dom-content-loaded-vs-load/
+      await this.page.reload({ waitUntil: "load" })
+    }
+  }
+
+  /**
+   * Checks if the editor is visible
+   */
+  async isEditorVisible() {
+    // Make sure the editor actually loaded
+    // If it's not visible after 5 seconds, something is wrong
+    await this.page.waitForLoadState("networkidle")
+    return await this.page.isVisible("div.monaco-workbench", { timeout: 5000 })
+  }
+
+  /**
+   * Focuses Integrated Terminal
+   * by going to the Application Menu
+   * and clicking View > Terminal
+   */
+  async focusTerminal() {
+    // If the terminal is already visible
+    // then we can focus it by hitting the keyboard shortcut
+    const isTerminalVisible = await this.page.isVisible("#terminal")
+    if (isTerminalVisible) {
+      await this.page.keyboard.press(`Control+Backquote`)
+      // Wait for terminal to receive focus
+      await this.page.waitForSelector("div.terminal.xterm.focus")
+      // Sometimes the terminal reloads
+      // which is why we wait for it twice
+      await this.page.waitForSelector("div.terminal.xterm.focus")
+      return
+    }
+    // Open using the manu
+    // Click [aria-label="Application Menu"] div[role="none"]
+    await this.page.click('[aria-label="Application Menu"] div[role="none"]')
+
+    // Click text=View
+    await this.page.hover("text=View")
+    await this.page.click("text=View")
+
+    // Click text=Terminal
+    await this.page.hover("text=Terminal")
+    await this.page.click("text=Terminal")
+
+    // Wait for terminal to receive focus
+    // Sometimes the terminal reloads once or twice
+    // which is why we wait for it to have the focus class
+    await this.page.waitForSelector("div.terminal.xterm.focus")
+    // Sometimes the terminal reloads
+    // which is why we wait for it twice
+    await this.page.waitForSelector("div.terminal.xterm.focus")
+  }
+
+  /**
+   * Navigates to CODE_SERVER_ADDRESS
+   * and reloads until the editor is visible
+   *
+   * Helpful for running before tests
+   */
+  async setup() {
+    await this.navigate()
+    await this.reloadUntilEditorIsVisible()
+  }
+}