refactor: move integration tests to Jest

.github/workflows/ci.yaml

@@ -288,8 +288,11 @@ jobs:
       - name: Build standalone release
         run: source scl_source enable devtoolset-9 && yarn release:standalone
 
-      - name: Sanity test standalone release
-        run: yarn test:standalone-release
+      - name: Install test dependencies
+        run: SKIP_SUBMODULE_DEPS=1 yarn install
+
+      - name: Run integration tests on standalone release
+        run: yarn test:integration
 
       - name: Build packages with nfpm
         run: yarn package
@@ -421,8 +424,11 @@ jobs:
       - name: Build standalone release
         run: yarn release:standalone
 
-      - name: Sanity test standalone release
-        run: yarn test:standalone-release
+      - name: Install test dependencies
+        run: SKIP_SUBMODULE_DEPS=1 yarn install
+
+      - name: Run integration tests on standalone release
+        run: yarn test:integration
 
       - name: Build packages with nfpm
         run: yarn package

.github/workflows/installer.yml

@@ -34,7 +34,7 @@ jobs:
         run: ./install.sh
 
       - name: Test code-server
-        run: yarn test:standalone-release code-server
+        run: CODE_SERVER_PATH="code-server" yarn test:integration
 
   alpine:
     name: Test installer on Alpine
@@ -66,4 +66,4 @@ jobs:
         run: ./install.sh
 
       - name: Test code-server
-        run: yarn test:standalone-release code-server
+        run: CODE_SERVER_PATH="code-server" yarn test:integration

ci/README.md

@@ -45,9 +45,6 @@ You can disable minification by setting `MINIFY=`.
   - Builds vscode into `./lib/vscode/out-vscode`.
 - [./ci/build/build-release.sh](./build/build-release.sh) (`yarn release`)
   - Bundles the output of the above two scripts into a single node module at `./release`.
-- [./ci/build/build-standalone-release.sh](./build/build-standalone-release.sh) (`yarn release:standalone`)
-  - Requires a node module already built into `./release` with the above script.
-  - Will build a standalone release with node and node_modules bundled into `./release-standalone`.
 - [./ci/build/clean.sh](./build/clean.sh) (`yarn clean`)
   - Removes all build artifacts.
   - Useful to do a clean build.
@@ -97,6 +94,8 @@ Helps avoid clobbering the CI configuration.
   - Runs `yarn lint`.
 - [./steps/test-unit.sh](./steps/test-unit.sh)
   - Runs `yarn test:unit`.
+- [./steps/test-integration.sh](./steps/test-integration.sh)
+  - Runs `yarn test:integration`.
 - [./steps/test-e2e.sh](./steps/test-e2e.sh)
   - Runs `yarn test:e2e`.
 - [./steps/release.sh](./steps/release.sh)

ci/dev/postinstall.sh

@@ -29,7 +29,11 @@ main() {
 
   install-deps test
   install-deps test/e2e/extensions/test-extension
-  install-deps lib/vscode
+  # We don't need these when running the integration tests
+  # so you can pass SKIP_SUBMODULE_DEPS
+  if [[ ! ${SKIP_SUBMODULE_DEPS-} ]]; then
+    install-deps lib/vscode
+  fi
 }
 
 main "$@"

ci/dev/test-integration.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+help() {
+  echo >&2 "  You can build the standalone release with 'yarn release:standalone'"
+  echo >&2 "  Or you can pass in a custom path."
+  echo >&2 "  CODE_SERVER_PATH='/var/tmp/coder/code-server/bin/code-server' yarn test:integration"
+}
+
+# Make sure a code-server release works. You can pass in the path otherwise it
+# will look for release-standalone in the current directory.
+#
+# This is to make sure we don't have Node version errors or any other
+# compilation-related errors.
+main() {
+  cd "$(dirname "$0")/../.."
+
+  source ./ci/lib.sh
+
+  local path="$RELEASE_PATH-standalone/bin/code-server"
+  if [[ ! ${CODE_SERVER_PATH-} ]]; then
+    echo "Set CODE_SERVER_PATH to test another build of code-server"
+  else
+    path="$CODE_SERVER_PATH"
+  fi
+
+  echo "Running tests with code-server binary: '$path'"
+
+  if [[ ! -f $path ]]; then
+    echo >&2 "No code-server build detected"
+    echo >&2 "Looked in $path"
+    help
+    exit 1
+  fi
+
+  CODE_SERVER_PATH="$path" CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --coverage=false --testRegex "./test/integration" --testPathIgnorePatterns "./test/integration/fixtures"
+}
+
+main "$@"

ci/dev/test-unit.sh

@@ -30,7 +30,7 @@ main() {
   # We must keep jest in a sub-directory. See ../../test/package.json for more
   # information. We must also run it from the root otherwise coverage will not
   # include our source files.
-  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@"
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --testRegex "./test/unit/.*ts" --testPathIgnorePatterns "./test/unit/node/test-plugin"
 }
 
 main "$@"

docs/CONTRIBUTING.md

@@ -147,7 +147,7 @@ Build the release packages (make sure that you run `yarn release` first):
 
 ```shell
 yarn release:standalone
-yarn test:standalone-release
+yarn test:integration
 yarn package
 ```
 
@@ -188,9 +188,8 @@ We use these to test anything related to our scripts (most of which live under `
 
 ### Integration tests
 
-These are a work in progress. We build code-server and run a script called
-[test-standalone-release.sh](../ci/build/test-standalone-release.sh), which
-ensures that code-server's CLI is working.
+These are a work in progress. We build code-server and run tests with `yarn test:integration`, which ensures that code-server builds work on their respective
+platforms.
 
 Our integration tests look at components that rely on one another. For example,
 testing the CLI requires us to build and package code-server.

package.json

@@ -18,8 +18,8 @@
     "release:github-assets": "./ci/build/release-github-assets.sh",
     "release:prep": "./ci/build/release-prep.sh",
     "test:e2e": "VSCODE_IPC_HOOK_CLI= ./ci/dev/test-e2e.sh",
-    "test:standalone-release": "./ci/build/test-standalone-release.sh",
     "test:unit": "./ci/dev/test-unit.sh --forceExit --detectOpenHandles",
+    "test:integration": "./ci/dev/test-integration.sh",
     "test:scripts": "./ci/dev/test-scripts.sh",
     "package": "./ci/build/build-packages.sh",
     "postinstall": "./ci/dev/postinstall.sh",

test/integration/installExtension.test.ts

@@ -0,0 +1,25 @@
+import { stat } from "fs/promises"
+import path from "path"
+import { clean, tmpdir } from "../utils/helpers"
+import { runCodeServerCommand } from "../utils/runCodeServerCommand"
+
+describe("--install-extension", () => {
+  const testName = "installExtension"
+  let tempDir: string
+  let setupFlags: string[]
+
+  beforeEach(async () => {
+    await clean(testName)
+    tempDir = await tmpdir(testName)
+    setupFlags = ["--extensions-dir", tempDir]
+  })
+  it("should install an extension", async () => {
+    const extName = `wesbos.theme-cobalt2-2.1.6`
+    const vsixFileName = "wesbos.theme-cobalt2-2.1.6.vsix"
+    const extensionFixture = path.resolve(`./test/integration/fixtures/${vsixFileName}`)
+    await runCodeServerCommand([...setupFlags, "--install-extension", extensionFixture])
+    const pathToExtFolder = path.join(tempDir, extName)
+    const statInfo = await stat(pathToExtFolder)
+    expect(statInfo.isDirectory()).toBe(true)
+  }, 20000)
+})

test/integration/listExtensions.test.ts

@@ -0,0 +1,30 @@
+import { rename } from "fs/promises"
+import path from "path"
+import extract from "extract-zip"
+import { clean, tmpdir } from "../utils/helpers"
+import { runCodeServerCommand } from "../utils/runCodeServerCommand"
+
+describe("--list-extensions", () => {
+  const testName = "listExtensions"
+  const extName = `wesbos.theme-cobalt2`
+  const extVersion = "2.1.6"
+  const vsixFileName = `${extName}-${extVersion}.vsix`
+  let tempDir: string
+  let setupFlags: string[]
+
+  beforeEach(async () => {
+    await clean(testName)
+    tempDir = await tmpdir(testName)
+    setupFlags = ["--extensions-dir", tempDir]
+    const extensionFixture = path.resolve(`./test/integration/fixtures/${vsixFileName}`)
+    // Make folder because this is where we'll move the extension
+    const pathToUnpackedExtension = path.join(tempDir, `${extName}-${extVersion}`)
+    const tempPathToUnpackedExtension = path.join(tempDir, `${extName}-temp`)
+    await extract(extensionFixture, { dir: tempPathToUnpackedExtension })
+    await rename(path.join(tempPathToUnpackedExtension, "extension", pathToUnpackedExtension))
+  })
+  it("should list installed extensions", async () => {
+    const { stdout } = await runCodeServerCommand([...setupFlags, "--list-extensions"])
+    expect(stdout).toMatch(extName)
+  }, 20000)
+})

test/package.json

@@ -10,6 +10,7 @@
     "@types/supertest": "^2.0.11",
     "@types/wtfnode": "^0.7.0",
     "argon2": "^0.28.0",
+    "extract-zip": "^2.0.1",
     "jest": "^27.3.1",
     "jest-fetch-mock": "^3.0.3",
     "jsdom": "^16.4.0",

test/utils/runCodeServerCommand.ts

@@ -0,0 +1,14 @@
+import { exec } from "child_process"
+import path from "path"
+import { promisify } from "util"
+
+/**
+ *
+ * A helper function for integration tests to run code-server commands.
+ */
+export async function runCodeServerCommand(argv: string[]): Promise<{ stdout: string; stderr: string }> {
+  const CODE_SERVER_COMMAND = process.env.CODE_SERVER_PATH || path.resolve("../../release-standalone/bin/code-server")
+  const { stdout, stderr } = await promisify(exec)(`${CODE_SERVER_COMMAND} ${argv.join(" ")}`)
+
+  return { stdout, stderr }
+}

test/yarn.lock

@@ -1690,9 +1690,9 @@ globals@^11.1.0:
   integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==
 
 graceful-fs@^4.2.4:
-  version "4.2.9"
-  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96"
-  integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==
+  version "4.2.10"
+  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c"
+  integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==
 
 has-flag@^3.0.0:
   version "3.0.0"