Move Code to a submodule

Closes #4901.

Author: code-asher

.github/workflows/ci.yaml

@@ -29,6 +29,9 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -71,6 +74,9 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -102,9 +108,11 @@ jobs:
     env:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     steps:
+      - name: Checkout repo
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -119,7 +127,7 @@ jobs:
       #     path: |
       #       "**/node_modules"
       #       "**/vendor/modules"
-      #       "**/vendor/modules/code-oss-dev/node_modules"
+      #       "**/lib/vscode/node_modules"
       #     key: yarn-build-${{ hashFiles('**/yarn.lock') }}-${{ hashFiles('**/vendor/yarn.lock') }}
       #     restore-keys: |
       #       yarn-build-
@@ -131,27 +139,26 @@ jobs:
       - name: Build code-server
         run: yarn build
 
-      # Parse the hash of the latest commit inside vendor/modules/code-oss-dev
+      # Parse the hash of the latest commit inside lib/vscode
       # use this to avoid rebuilding it if nothing changed
       # How it works: the `git log` command fetches the hash of the last commit
-      # that changed a file inside `vendor/modules/code-oss-dev`. If a commit changes any file in there,
+      # that changed a file inside `lib/vscode`. If a commit changes any file in there,
       # the hash returned will change, and we rebuild vscode. If the hash did not change,
       # (for example, a change to `src/` or `docs/`), we reuse the same build as last time.
       # This saves a lot of time in CI, as compiling VSCode can take anywhere from 5-10 minutes.
-      - name: Get latest vendor/modules/code-oss-dev rev
+      - name: Get latest lib/vscode rev
         id: vscode-rev
-        run: echo "::set-output name=rev::$(jq -r '.devDependencies["code-oss-dev"]' vendor/package.json | sed -r 's|.*#(.*)$|\1|')"
+        run: echo "::set-output name=rev::$(jq -r '.devDependencies["code-oss-dev"]' lib/vscode/package.json | sed -r 's|.*#(.*)$|\1|')"
 
       - name: Attempt to fetch vscode build from cache
         id: cache-vscode-2
         uses: actions/cache@v2
         with:
           path: |
-            vendor/modules/code-oss-dev/.build
-            vendor/modules/code-oss-dev/package.json
-            vendor/modules/code-oss-dev/out-build
-            vendor/modules/code-oss-dev/out-vscode-reh-web
-            vendor/modules/code-oss-dev/out-vscode-reh-web-min
+            lib/vscode/.build
+            lib/vscode/out-build
+            lib/vscode/out-vscode-reh-web
+            lib/vscode/out-vscode-reh-web-min
           key: vscode-reh-build-${{ secrets.VSCODE_CACHE_VERSION }}-${{ steps.vscode-rev.outputs.rev }}
 
       - name: Build vscode
@@ -197,7 +204,11 @@ jobs:
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout repo
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - uses: actions/download-artifact@v3
         id: download
@@ -226,7 +237,11 @@ jobs:
     container: "centos:7"
 
     steps:
+      - name: Checkout repo
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -315,7 +330,11 @@ jobs:
       NODE_VERSION: v14.17.4
 
     steps:
+      - name: Checkout repo
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -364,7 +383,11 @@ jobs:
     runs-on: macos-latest
     timeout-minutes: 15
     steps:
+      - name: Checkout repo
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -409,7 +432,11 @@ jobs:
       # since VS Code will load faster due to the bundling.
       CODE_SERVER_TEST_ENTRY: "./release-packages/code-server-linux-amd64"
     steps:
+      - name: Checkout repo
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
       - name: Install Node.js v14
         uses: actions/setup-node@v3
@@ -446,12 +473,6 @@ jobs:
           ./test/node_modules/.bin/playwright install-deps
           ./test/node_modules/.bin/playwright install
 
-      # TODO@jsjoeio - remove once we switch to submodules.
-      - name: Create package.json for testing
-        run: |
-          mkdir -p ./vendor/modules/code-oss-dev
-          echo '{ "version": "test" }' > ./vendor/modules/code-oss-dev/package.json
-
       - name: Run end-to-end tests
         run: yarn test:e2e
 
@@ -470,6 +491,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
+
       - name: Run Trivy vulnerability scanner in repo mode
         uses: aquasecurity/trivy-action@296212627a1e693efa09c00adc3e03b2ba8edf18
         with:
@@ -480,6 +505,7 @@ jobs:
           template: "@/contrib/sarif.tpl"
           output: "trivy-repo-results.sarif"
           severity: "HIGH,CRITICAL"
+
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v1
         with:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lib/vscode"]
+	path = lib/vscode
+	url = https://github.com/coder/vscode

.prettierrc.yaml

@@ -7,7 +7,7 @@ useTabs: false
 
 overrides:
   # Attempt to keep VScode's existing code style intact.
-  - files: "vendor/modules/code-oss-dev/**/*.ts"
+  - files: "lib/vscode/**/*.ts"
     options:
       # No limit defined upstream.
       printWidth: 10000

.tours/contributing.tour

@@ -143,7 +143,7 @@
       "description": "Static images and the manifest live here in `src/browser/media` (see the explorer)."
     },
     {
-      "directory": "vendor/modules/code-oss-dev",
+      "directory": "lib/vscode",
       "line": 1,
       "description": "code-server makes use of VS Code's frontend web/remote support. Most of the modifications implement the remote server since that portion of the code is closed source and not released with VS Code.\n\nWe also have a few bug fixes and have added some features (like client-side extensions). See [https://github.com/coder/code-server/blob/master/docs/CONTRIBUTING.md#modifications-to-vs-code](https://github.com/coder/code-server/blob/master/docs/CONTRIBUTING.md#modifications-to-vs-code) for a list.\n\nWe make an effort to keep the modifications as few as possible."
     }

ci/build/build-release.sh

@@ -15,8 +15,8 @@ main() {
 
   source ./ci/lib.sh
 
-  VSCODE_SRC_PATH="vendor/modules/code-oss-dev"
-  VSCODE_OUT_PATH="$RELEASE_PATH/vendor/modules/code-oss-dev"
+  VSCODE_SRC_PATH="lib/vscode"
+  VSCODE_OUT_PATH="$RELEASE_PATH/lib/vscode"
 
   mkdir -p "$RELEASE_PATH"
 
@@ -25,7 +25,7 @@ main() {
 
   rsync ./docs/README.md "$RELEASE_PATH"
   rsync LICENSE.txt "$RELEASE_PATH"
-  rsync ./vendor/modules/code-oss-dev/ThirdPartyNotices.txt "$RELEASE_PATH"
+  rsync ./lib/vscode/ThirdPartyNotices.txt "$RELEASE_PATH"
 }
 
 bundle_code_server() {

ci/build/build-standalone-release.sh

@@ -34,7 +34,7 @@ main() {
   # leaves a few stray symlinks. Clean them up so nfpm does not fail.
   # Remove this line when its no longer needed.
 
-  rm -fr "$RELEASE_PATH/vendor/modules/code-oss-dev/extensions/node_modules/.bin"
+  rm -fr "$RELEASE_PATH/lib/vscode/extensions/node_modules/.bin"
 }
 
 main "$@"

ci/build/build-vscode.sh

@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Builds vscode into vendor/modules/code-oss-dev/out-vscode.
+# Builds vscode into lib/vscode/out-vscode.
 
 # MINIFY controls whether a minified version of vscode is built.
 MINIFY=${MINIFY-true}
 
 main() {
   cd "$(dirname "${0}")/../.."
 
-  cd vendor/modules/code-oss-dev
+  cd lib/vscode
 
   # Any platform works since we have our own packaging step (for now).
   yarn gulp "vscode-reh-web-linux-x64${MINIFY:+-min}"

ci/build/npm-postinstall.sh

@@ -90,8 +90,8 @@ symlink_asar() {
 }
 
 vscode_yarn() {
-  echo 'Installing vendor dependencies...'
-  cd vendor/modules/code-oss-dev
+  echo 'Installing Code dependencies...'
+  cd lib/vscode
   yarn --production --frozen-lockfile
 
   symlink_asar

ci/dev/fmt.sh

@@ -19,7 +19,7 @@ main() {
     "*.sh"
   )
   prettier --write --loglevel=warn $(
-    git ls-files "${prettierExts[@]}" | grep -v "lib/vscode" | grep -v "vendor/modules/code-oss-dev" | grep -v 'helm-chart'
+    git ls-files "${prettierExts[@]}" | grep -v "lib/vscode" | grep -v 'helm-chart'
   )
 
   doctoc --title '# FAQ' docs/FAQ.md > /dev/null

ci/dev/lint.sh

@@ -4,10 +4,10 @@ set -euo pipefail
 main() {
   cd "$(dirname "$0")/../.."
 
-  eslint --max-warnings=0 --fix $(git ls-files "*.ts" "*.tsx" "*.js" | grep -v "vendor/modules/code-oss-dev" | grep -v "lib/vscode")
-  stylelint $(git ls-files "*.css" | grep -v "vendor/modules/code-oss-dev" | grep -v "lib/vscode")
+  eslint --max-warnings=0 --fix $(git ls-files "*.ts" "*.tsx" "*.js" | grep -v "lib/vscode")
+  stylelint $(git ls-files "*.css" | grep -v "lib/vscode")
   tsc --noEmit --skipLibCheck
-  shellcheck -e SC2046,SC2164,SC2154,SC1091,SC1090,SC2002 $(git ls-files "*.sh" | grep -v "vendor/modules/code-oss-dev" | grep -v "lib/vscode")
+  shellcheck -e SC2046,SC2164,SC2154,SC1091,SC1090,SC2002 $(git ls-files "*.sh" | grep -v "lib/vscode")
   if command -v helm && helm kubeval --help > /dev/null; then
     helm kubeval ci/helm-chart
   fi

ci/dev/postinstall.sh

@@ -1,50 +1,25 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-main() {
-  cd "$(dirname "$0")/../.."
-  source ./ci/lib.sh
-
-  pushd test
-  echo "Installing dependencies for $PWD"
-  yarn install
-  popd
-
+# Install dependencies in $1.
+install-deps() {
   local args=(install)
   if [[ ${CI-} ]]; then
     args+=(--frozen-lockfile)
   fi
-
-  pushd test
-  echo "Installing dependencies for $PWD"
-  yarn "${args[@]}"
-  popd
-
-  pushd test/e2e/extensions/test-extension
-  echo "Installing dependencies for $PWD"
-  yarn "${args[@]}"
+  pushd "$1"
+    echo "Installing dependencies for $PWD"
+    yarn "${args[@]}"
   popd
+}
 
-  pushd vendor
-  echo "Installing dependencies for $PWD"
-
-  # We install in 'modules' instead of 'node_modules' because VS Code's
-  # extensions use a webpack config which cannot differentiate between its own
-  # node_modules and itself being in a directory with the same name.
-  args+=(--modules-folder modules)
-
-  # We ignore scripts because NPM/Yarn's default behavior is to assume that
-  # devDependencies are not needed, and that even git repo based packages are
-  # assumed to be compiled. Because the default behavior for VS Code's
-  # `postinstall` assumes we're also compiled, this needs to be ignored.
-  args+=(--ignore-scripts)
-
-  yarn "${args[@]}"
-
-  # Finally, run the vendor `postinstall`
-  yarn run postinstall
+main() {
+  cd "$(dirname "$0")/../.."
+  source ./ci/lib.sh
 
-  popd
+  install-deps test
+  install-deps test/e2e/extensions/test-extension
+  install-deps lib/vscode
 }
 
 main "$@"

ci/dev/test-e2e.sh

@@ -37,7 +37,7 @@ main() {
     exit 1
   fi
 
-  if [[ ! -d $dir/vendor/modules/code-oss-dev/out ]]; then
+  if [[ ! -d $dir/lib/vscode/out ]]; then
     echo >&2 "No VS Code build detected"
     help
     exit 1

ci/dev/test-unit.sh

@@ -14,7 +14,7 @@ main() {
   # Our code imports from `out` in order to work during development but if you
   # have only built for production you will have not have this directory.  In
   # that case symlink `out` to a production build directory.
-  local vscode="vendor/modules/code-oss-dev"
+  local vscode="lib/vscode"
   local link="$vscode/out"
   local target="out-build"
   if [[ ! -e $link ]] && [[ -d $vscode/$target ]]; then

ci/dev/watch.ts

@@ -14,7 +14,7 @@ class Watcher {
   private rootPath = path.resolve(process.cwd())
   private readonly paths = {
     /** Path to uncompiled VS Code source. */
-    vscodeDir: path.join(this.rootPath, "vendor", "modules", "code-oss-dev"),
+    vscodeDir: path.join(this.rootPath, "lib/vscode"),
     pluginDir: process.env.PLUGIN_DIR,
   }
 

ci/lib.sh

@@ -14,7 +14,7 @@ pkg_json_version() {
 }
 
 vscode_version() {
-  jq -r .version vendor/modules/code-oss-dev/package.json
+  jq -r .version lib/vscode/package.json
 }
 
 os() {

docs/CONTRIBUTING.md

@@ -127,7 +127,6 @@
     "testEnvironment": "node",
     "testPathIgnorePatterns": [
       "/node_modules/",
-      "/vendor/",
       "/lib/",
       "/out/",
       "test/e2e"
@@ -158,7 +157,7 @@
       "<rootDir>/release-npm-package",
       "<rootDir>/release-gcp",
       "<rootDir>/release-images",
-      "<rootDir>/vendor"
+      "<rootDir>/lib"
     ],
     "moduleNameMapper": {
       "^.+\\.(css|less)$": "<rootDir>/test/utils/cssStub.ts"

lib/vscode

@@ -17,7 +17,7 @@ export function getPackageJson(relativePath: string): JSONSchemaForNPMPackageJso
 }
 
 export const rootPath = path.resolve(__dirname, "../..")
-export const vsRootPath = path.join(rootPath, "vendor/modules/code-oss-dev")
+export const vsRootPath = path.join(rootPath, "lib/vscode")
 const PACKAGE_JSON = "package.json"
 const pkg = getPackageJson(`${rootPath}/${PACKAGE_JSON}`)
 const codePkg = getPackageJson(`${vsRootPath}/${PACKAGE_JSON}`) || { version: "0.0.0" }

package.json

@@ -31,7 +31,7 @@ export const shouldSpawnCliProcess = (args: UserProvidedArgs): boolean => {
 export const runVsCodeCli = async (args: DefaultedArgs): Promise<void> => {
   logger.debug("Running VS Code CLI")
 
-  // See ../../vendor/modules/code-oss-dev/src/vs/server/node/server.main.js.
+  // See ../../lib/vscode/src/vs/server/node/server.main.js.
   const spawnCli = await loadAMDModule<CodeServerLib.SpawnCli>("vs/server/node/server.main", "spawnCli")
 
   try {

src/node/constants.ts

@@ -118,7 +118,7 @@ export class CodeServerRouteWrapper {
     const { args } = req
 
     /**
-     * @file ../../../vendor/modules/code-oss-dev/src/vs/server/node/server.main.js
+     * @file ../../../lib/vscode/src/vs/server/node/server.main.js
      */
     const createVSServer = await loadAMDModule<CodeServerLib.CreateServer>("vs/server/node/server.main", "createServer")
 

src/node/main.ts

@@ -16,7 +16,7 @@ describe("constants", () => {
     }
 
     const mockCodePackageJson = {
-      name: "mock-code-oss-dev",
+      name: "mock-vscode",
       version: "1.2.3",
     }
 
@@ -25,7 +25,7 @@ describe("constants", () => {
       mockLogger()
       jest.mock(path.resolve(__dirname, "../../../package.json"), () => mockPackageJson, { virtual: true })
       jest.mock(
-        path.resolve(__dirname, "../../../vendor/modules/code-oss-dev/package.json"),
+        path.resolve(__dirname, "../../../lib/vscode/package.json"),
         () => mockCodePackageJson,
         { virtual: true },
       )
@@ -96,7 +96,7 @@ describe("constants", () => {
         const packageJson = constants.getPackageJson("../../package.json")
         expect(packageJson).toStrictEqual(mockPackageJson)
 
-        const codePackageJson = constants.getPackageJson("../../vendor/modules/code-oss-dev/package.json")
+        const codePackageJson = constants.getPackageJson("../../lib/vscode/package.json")
         expect(codePackageJson).toStrictEqual(mockCodePackageJson)
       })
     })
@@ -107,14 +107,14 @@ describe("constants", () => {
       name: "mock-code-server",
     }
     const mockCodePackageJson = {
-      name: "mock-code-oss-dev",
+      name: "mock-vscode",
     }
 
     beforeAll(() => {
       jest.clearAllMocks()
       jest.mock(path.resolve(__dirname, "../../../package.json"), () => mockPackageJson, { virtual: true })
       jest.mock(
-        path.resolve(__dirname, "../../../vendor/modules/code-oss-dev/package.json"),
+        path.resolve(__dirname, "../../../lib/vscode/package.json"),
         () => mockCodePackageJson,
         { virtual: true },
       )

src/node/routes/vscode.ts

@@ -20,7 +20,7 @@
       "./node_modules/@types",
       "./typings",
       "./test/node_modules/@types",
-      "./vendor/modules/code-oss-dev/src/vs/server/@types"
+      "./lib/vscode/src/vs/server/@types"
     ],
     "downlevelIteration": true
   },