From 613dcbe397f3ffba39bcd26d302f783960b0fc76 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Fri, 23 Sep 2022 14:37:16 -0700 Subject: [PATCH 01/20] refactor: move platform steps build -> release --- .github/workflows/build.yaml | 224 --------------------------------- .github/workflows/release.yaml | 224 +++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 224 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8d8afb5c850f..b41738c071a7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -274,230 +274,6 @@ jobs: npm install -g @coder/code-server-pr@${{ github.event.number }} ``` - # TODO: cache building yarn --production - # possibly 2m30s of savings(?) - # this requires refactoring our release scripts - package-linux-amd64: - name: x86-64 Linux build - needs: build - runs-on: ubuntu-latest - timeout-minutes: 15 - container: "centos:7" - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Install Node.js v16 - uses: actions/setup-node@v3 - with: - node-version: "16" - - - name: Install development tools - run: | - yum install -y epel-release centos-release-scl make - yum install -y devtoolset-9-{make,gcc,gcc-c++} jq rsync python3 - - - name: Install nfpm and envsubst - run: | - mkdir -p ~/.local/bin - curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm - curl -sSfL https://github.com/a8m/envsubst/releases/download/v1.1.0/envsubst-`uname -s`-`uname -m` -o envsubst - chmod +x envsubst - mv envsubst ~/.local/bin - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Install yarn - run: npm install -g yarn - - - name: Download npm package - uses: actions/download-artifact@v3 - with: - name: npm-package - - - name: Decompress npm package - run: tar -xzf package.tar.gz - - # NOTE: && here is deliberate - GitHub puts each line in its own `.sh` - # file when running inside a docker container. - - name: Build standalone release - run: source scl_source enable devtoolset-9 && yarn release:standalone - - - name: Fetch dependencies from cache - id: cache-node-modules - uses: actions/cache@v3 - with: - path: "**/node_modules" - key: yarn-build-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn-build- - - - name: Install test dependencies - if: steps.cache-node-modules.outputs.cache-hit != 'true' - run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile - - - name: Run integration tests on standalone release - run: yarn test:integration - - - name: Upload coverage report to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - if: success() - - - name: Build packages with nfpm - run: yarn package - - - name: Upload release artifacts - uses: actions/upload-artifact@v3 - with: - name: release-packages - path: ./release-packages - - # NOTE@oxy: - # We use Ubuntu 16.04 here, so that our build is more compatible - # with older libc versions. We used to (Q1'20) use CentOS 7 here, - # but it has a full update EOL of Q4'20 and a 'critical security' - # update EOL of 2024. We're dropping full support a few years before - # the final EOL, but I don't believe CentOS 7 has a large arm64 userbase. - # It is not feasible to cross-compile with CentOS. - - # Cross-compile notes: To compile native dependencies for arm64, - # we install the aarch64/armv7l cross toolchain and then set it as the default - # compiler/linker/etc. with the AR/CC/CXX/LINK environment variables. - # qemu-user-static on ubuntu-16.04 currently doesn't run Node correctly, - # so we just build with "native"/x86_64 node, then download arm64/armv7l node - # and then put it in our release. We can't smoke test the cross build this way, - # but this means we don't need to maintain a self-hosted runner! - - # NOTE@jsjoeio: - # We used to use 16.04 until GitHub deprecated it on September 20, 2021 - # See here: https://github.com/actions/virtual-environments/pull/3862/files - package-linux-cross: - name: Linux cross-compile builds - needs: build - runs-on: ubuntu-18.04 - timeout-minutes: 15 - strategy: - matrix: - include: - - prefix: aarch64-linux-gnu - arch: arm64 - - prefix: arm-linux-gnueabihf - arch: armv7l - - env: - AR: ${{ format('{0}-ar', matrix.prefix) }} - CC: ${{ format('{0}-gcc', matrix.prefix) }} - CXX: ${{ format('{0}-g++', matrix.prefix) }} - LINK: ${{ format('{0}-g++', matrix.prefix) }} - NPM_CONFIG_ARCH: ${{ matrix.arch }} - NODE_VERSION: v16.13.0 - - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Install Node.js v16 - uses: actions/setup-node@v3 - with: - node-version: "16" - - - name: Install nfpm - run: | - mkdir -p ~/.local/bin - curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Install cross-compiler - run: sudo apt update && sudo apt install $PACKAGE - env: - PACKAGE: ${{ format('g++-{0}', matrix.prefix) }} - - - name: Download npm package - uses: actions/download-artifact@v3 - with: - name: npm-package - - - name: Decompress npm package - run: tar -xzf package.tar.gz - - - name: Build standalone release - run: yarn release:standalone - - - name: Replace node with cross-compile equivalent - run: | - wget https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}.tar.xz - tar -xf node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}.tar.xz node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}/bin/node --strip-components=2 - mv ./node ./release-standalone/lib/node - - - name: Build packages with nfpm - run: yarn package ${NPM_CONFIG_ARCH} - - - name: Upload release artifacts - uses: actions/upload-artifact@v3 - with: - name: release-packages - path: ./release-packages - - package-macos-amd64: - name: x86-64 macOS build - needs: build - runs-on: macos-latest - timeout-minutes: 15 - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Install Node.js v16 - uses: actions/setup-node@v3 - with: - node-version: "16" - - - name: Install nfpm - run: | - mkdir -p ~/.local/bin - curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Download npm package - uses: actions/download-artifact@v3 - with: - name: npm-package - - - name: Decompress npm package - run: tar -xzf package.tar.gz - - - name: Build standalone release - run: yarn release:standalone - - - name: Fetch dependencies from cache - id: cache-node-modules - uses: actions/cache@v3 - with: - path: "**/node_modules" - key: yarn-build-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn-build- - - - name: Install test dependencies - if: steps.cache-node-modules.outputs.cache-hit != 'true' - run: SKIP_SUBMODULE_DEPS=1 yarn install - - - name: Run native module tests on standalone release - run: yarn test:native - - - name: Build packages with nfpm - run: yarn package - - - name: Upload release artifacts - uses: actions/upload-artifact@v3 - with: - name: release-packages - path: ./release-packages - test-e2e: name: End-to-end tests needs: package-linux-amd64 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 181b98580a6c..08cd974ae634 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,6 +14,230 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + # TODO: cache building yarn --production + # possibly 2m30s of savings(?) + # this requires refactoring our release scripts + package-linux-amd64: + name: x86-64 Linux build + needs: build + runs-on: ubuntu-latest + timeout-minutes: 15 + container: "centos:7" + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Install Node.js v16 + uses: actions/setup-node@v3 + with: + node-version: "16" + + - name: Install development tools + run: | + yum install -y epel-release centos-release-scl make + yum install -y devtoolset-9-{make,gcc,gcc-c++} jq rsync python3 + + - name: Install nfpm and envsubst + run: | + mkdir -p ~/.local/bin + curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm + curl -sSfL https://github.com/a8m/envsubst/releases/download/v1.1.0/envsubst-`uname -s`-`uname -m` -o envsubst + chmod +x envsubst + mv envsubst ~/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install yarn + run: npm install -g yarn + + - name: Download npm package + uses: actions/download-artifact@v3 + with: + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + # NOTE: && here is deliberate - GitHub puts each line in its own `.sh` + # file when running inside a docker container. + - name: Build standalone release + run: source scl_source enable devtoolset-9 && yarn release:standalone + + - name: Fetch dependencies from cache + id: cache-node-modules + uses: actions/cache@v3 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn-build- + + - name: Install test dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile + + - name: Run integration tests on standalone release + run: yarn test:integration + + - name: Upload coverage report to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + if: success() + + - name: Build packages with nfpm + run: yarn package + + - name: Upload release artifacts + uses: actions/upload-artifact@v3 + with: + name: release-packages + path: ./release-packages + + # NOTE@oxy: + # We use Ubuntu 16.04 here, so that our build is more compatible + # with older libc versions. We used to (Q1'20) use CentOS 7 here, + # but it has a full update EOL of Q4'20 and a 'critical security' + # update EOL of 2024. We're dropping full support a few years before + # the final EOL, but I don't believe CentOS 7 has a large arm64 userbase. + # It is not feasible to cross-compile with CentOS. + + # Cross-compile notes: To compile native dependencies for arm64, + # we install the aarch64/armv7l cross toolchain and then set it as the default + # compiler/linker/etc. with the AR/CC/CXX/LINK environment variables. + # qemu-user-static on ubuntu-16.04 currently doesn't run Node correctly, + # so we just build with "native"/x86_64 node, then download arm64/armv7l node + # and then put it in our release. We can't smoke test the cross build this way, + # but this means we don't need to maintain a self-hosted runner! + + # NOTE@jsjoeio: + # We used to use 16.04 until GitHub deprecated it on September 20, 2021 + # See here: https://github.com/actions/virtual-environments/pull/3862/files + package-linux-cross: + name: Linux cross-compile builds + needs: build + runs-on: ubuntu-18.04 + timeout-minutes: 15 + strategy: + matrix: + include: + - prefix: aarch64-linux-gnu + arch: arm64 + - prefix: arm-linux-gnueabihf + arch: armv7l + + env: + AR: ${{ format('{0}-ar', matrix.prefix) }} + CC: ${{ format('{0}-gcc', matrix.prefix) }} + CXX: ${{ format('{0}-g++', matrix.prefix) }} + LINK: ${{ format('{0}-g++', matrix.prefix) }} + NPM_CONFIG_ARCH: ${{ matrix.arch }} + NODE_VERSION: v16.13.0 + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Install Node.js v16 + uses: actions/setup-node@v3 + with: + node-version: "16" + + - name: Install nfpm + run: | + mkdir -p ~/.local/bin + curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install cross-compiler + run: sudo apt update && sudo apt install $PACKAGE + env: + PACKAGE: ${{ format('g++-{0}', matrix.prefix) }} + + - name: Download npm package + uses: actions/download-artifact@v3 + with: + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Build standalone release + run: yarn release:standalone + + - name: Replace node with cross-compile equivalent + run: | + wget https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}.tar.xz + tar -xf node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}.tar.xz node-${NODE_VERSION}-linux-${NPM_CONFIG_ARCH}/bin/node --strip-components=2 + mv ./node ./release-standalone/lib/node + + - name: Build packages with nfpm + run: yarn package ${NPM_CONFIG_ARCH} + + - name: Upload release artifacts + uses: actions/upload-artifact@v3 + with: + name: release-packages + path: ./release-packages + + package-macos-amd64: + name: x86-64 macOS build + needs: build + runs-on: macos-latest + timeout-minutes: 15 + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Install Node.js v16 + uses: actions/setup-node@v3 + with: + node-version: "16" + + - name: Install nfpm + run: | + mkdir -p ~/.local/bin + curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Download npm package + uses: actions/download-artifact@v3 + with: + name: npm-package + + - name: Decompress npm package + run: tar -xzf package.tar.gz + + - name: Build standalone release + run: yarn release:standalone + + - name: Fetch dependencies from cache + id: cache-node-modules + uses: actions/cache@v3 + with: + path: "**/node_modules" + key: yarn-build-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn-build- + + - name: Install test dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: SKIP_SUBMODULE_DEPS=1 yarn install + + - name: Run native module tests on standalone release + run: yarn test:native + + - name: Build packages with nfpm + run: yarn package + + - name: Upload release artifacts + uses: actions/upload-artifact@v3 + with: + name: release-packages + path: ./release-packages + draft: runs-on: ubuntu-latest steps: From b35aa65ea7c932b393a18e7fa9a1921175355c79 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Fri, 23 Sep 2022 14:38:09 -0700 Subject: [PATCH 02/20] fixup! refactor: move platform steps build -> release --- .github/workflows/release.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 08cd974ae634..ef592c212b97 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,7 +19,6 @@ jobs: # this requires refactoring our release scripts package-linux-amd64: name: x86-64 Linux build - needs: build runs-on: ubuntu-latest timeout-minutes: 15 container: "centos:7" @@ -117,7 +116,6 @@ jobs: # See here: https://github.com/actions/virtual-environments/pull/3862/files package-linux-cross: name: Linux cross-compile builds - needs: build runs-on: ubuntu-18.04 timeout-minutes: 15 strategy: @@ -184,7 +182,6 @@ jobs: package-macos-amd64: name: x86-64 macOS build - needs: build runs-on: macos-latest timeout-minutes: 15 steps: From cc63e44be64db61e53d5add66fcaa297e3f8faff Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Fri, 23 Sep 2022 14:42:02 -0700 Subject: [PATCH 03/20] refactor: download npm package --- .github/workflows/release.yaml | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ef592c212b97..1aee685c568f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -51,10 +51,16 @@ jobs: - name: Install yarn run: npm install -g yarn - - name: Download npm package - uses: actions/download-artifact@v3 + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + id: download with: + branch: ${{ github.ref }} + workflow: build.yaml + workflow_conclusion: completed + check_artifacts: true name: npm-package + path: ./npm-package - name: Decompress npm package run: tar -xzf package.tar.gz @@ -154,10 +160,16 @@ jobs: env: PACKAGE: ${{ format('g++-{0}', matrix.prefix) }} - - name: Download npm package - uses: actions/download-artifact@v3 + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + id: download with: + branch: ${{ github.ref }} + workflow: build.yaml + workflow_conclusion: completed + check_artifacts: true name: npm-package + path: ./npm-package - name: Decompress npm package run: tar -xzf package.tar.gz @@ -199,10 +211,16 @@ jobs: curl -sSfL https://github.com/goreleaser/nfpm/releases/download/v2.3.1/nfpm_2.3.1_`uname -s`_`uname -m`.tar.gz | tar -C ~/.local/bin -zxv nfpm echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: Download npm package - uses: actions/download-artifact@v3 + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + id: download with: + branch: ${{ github.ref }} + workflow: build.yaml + workflow_conclusion: completed + check_artifacts: true name: npm-package + path: ./npm-package - name: Decompress npm package run: tar -xzf package.tar.gz From 8ea67d2c0b47a9846cf540f40099edf86afeaa5b Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Fri, 23 Sep 2022 14:44:50 -0700 Subject: [PATCH 04/20] refactor: upload release-packages to draft --- .github/workflows/release.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1aee685c568f..f0d22061bc5c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -95,11 +95,11 @@ jobs: - name: Build packages with nfpm run: yarn package - - name: Upload release artifacts - uses: actions/upload-artifact@v3 + - uses: softprops/action-gh-release@v1 with: - name: release-packages - path: ./release-packages + draft: true + discussion_category_name: "📣 Announcements" + files: ./release-packages/* # NOTE@oxy: # We use Ubuntu 16.04 here, so that our build is more compatible @@ -186,11 +186,11 @@ jobs: - name: Build packages with nfpm run: yarn package ${NPM_CONFIG_ARCH} - - name: Upload release artifacts - uses: actions/upload-artifact@v3 + - uses: softprops/action-gh-release@v1 with: - name: release-packages - path: ./release-packages + draft: true + discussion_category_name: "📣 Announcements" + files: ./release-packages/* package-macos-amd64: name: x86-64 macOS build @@ -247,11 +247,11 @@ jobs: - name: Build packages with nfpm run: yarn package - - name: Upload release artifacts - uses: actions/upload-artifact@v3 + - uses: softprops/action-gh-release@v1 with: - name: release-packages - path: ./release-packages + draft: true + discussion_category_name: "📣 Announcements" + files: ./release-packages/* draft: runs-on: ubuntu-latest From 038531896982966f211da00323408b50c92f5899 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Fri, 23 Sep 2022 16:04:02 -0700 Subject: [PATCH 05/20] refactor: remove draft step --- .github/workflows/release.yaml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f0d22061bc5c..58ce5903a8a7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -252,27 +252,3 @@ jobs: draft: true discussion_category_name: "📣 Announcements" files: ./release-packages/* - - draft: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Download artifacts - uses: dawidd6/action-download-artifact@v2 - id: download - with: - branch: ${{ github.ref }} - workflow: build.yaml - workflow_conclusion: completed - check_artifacts: true - name: release-packages - path: ./release-packages - - - uses: softprops/action-gh-release@v1 - with: - draft: true - discussion_category_name: "📣 Announcements" - files: ./release-packages/* From 7df770278a69076d32ba7251af61ace848afa06d Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Mon, 26 Sep 2022 13:16:24 -0700 Subject: [PATCH 06/20] refactor: e2e rely on build now --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b41738c071a7..03d620e76b4d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -276,7 +276,7 @@ jobs: test-e2e: name: End-to-end tests - needs: package-linux-amd64 + needs: build runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -337,7 +337,7 @@ jobs: test-e2e-proxy: name: End-to-end tests behind proxy - needs: package-linux-amd64 + needs: build runs-on: ubuntu-latest timeout-minutes: 25 env: From 002c0e6055ce87e9605ed5a0b589d9b24a642f23 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Mon, 26 Sep 2022 14:07:02 -0700 Subject: [PATCH 07/20] refactor: use npm package in e2e --- .github/workflows/build.yaml | 46 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 03d620e76b4d..26e7708ab5ed 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -301,29 +301,30 @@ jobs: restore-keys: | yarn-build- - - name: Download release packages + - name: Download npm package uses: actions/download-artifact@v3 with: - name: release-packages - path: ./release-packages + name: npm-package - - name: Untar code-server release - run: | - cd release-packages - tar -xzf code-server*-linux-amd64.tar.gz - mv code-server*-linux-amd64 code-server-linux-amd64 + - name: Decompress npm package + run: tar -xzf package.tar.gz - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' - run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile + run: yarn --frozen-lockfile - name: Install Playwright OS dependencies run: | ./test/node_modules/.bin/playwright install-deps ./test/node_modules/.bin/playwright install + - name: Copy dependencies to release (npm package) + run: | + cp -r node_modules release/ + cp -r lib/vscode/node_modules release/lib/vscode + - name: Run end-to-end tests - run: yarn test:e2e --global-timeout 840000 + run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 - name: Upload test artifacts if: always() @@ -333,7 +334,7 @@ jobs: path: ./test/test-results - name: Remove release packages and test artifacts - run: rm -rf ./release-packages ./test/test-results + run: rm -rf ./release ./test/test-results test-e2e-proxy: name: End-to-end tests behind proxy @@ -362,21 +363,17 @@ jobs: restore-keys: | yarn-build- - - name: Download release packages + - name: Download npm package uses: actions/download-artifact@v3 with: - name: release-packages - path: ./release-packages + name: npm-package - - name: Untar code-server release - run: | - cd release-packages - tar -xzf code-server*-linux-amd64.tar.gz - mv code-server*-linux-amd64 code-server-linux-amd64 + - name: Decompress npm package + run: tar -xzf package.tar.gz - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' - run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile + run: yarn --frozen-lockfile - name: Install Playwright OS dependencies run: | @@ -403,8 +400,13 @@ jobs: - name: Start Caddy run: sudo ~/.cache/caddy/caddy start --config ./ci/Caddyfile + - name: Copy dependencies to release (npm package) + run: | + cp -r node_modules release/ + cp -r lib/vscode/node_modules release/lib/vscode + - name: Run end-to-end tests - run: yarn test:e2e:proxy + run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 - name: Stop Caddy if: always() @@ -418,4 +420,4 @@ jobs: path: ./test/test-results - name: Remove release packages and test artifacts - run: rm -rf ./release-packages ./test/test-results + run: rm -rf ./release ./test/test-results From bd83b06b5881de0ff453cdb13efe15e674552d34 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Mon, 26 Sep 2022 14:08:26 -0700 Subject: [PATCH 08/20] fix: update release workflow --- .github/workflows/release.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 58ce5903a8a7..61ec8a5bae11 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -60,7 +60,6 @@ jobs: workflow_conclusion: completed check_artifacts: true name: npm-package - path: ./npm-package - name: Decompress npm package run: tar -xzf package.tar.gz @@ -169,7 +168,6 @@ jobs: workflow_conclusion: completed check_artifacts: true name: npm-package - path: ./npm-package - name: Decompress npm package run: tar -xzf package.tar.gz @@ -220,7 +218,6 @@ jobs: workflow_conclusion: completed check_artifacts: true name: npm-package - path: ./npm-package - name: Decompress npm package run: tar -xzf package.tar.gz From ac6e9c750f3154a9e4aefb5101840904c98a5834 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Mon, 26 Sep 2022 14:33:10 -0700 Subject: [PATCH 09/20] fixup: update cache key --- .github/workflows/build.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 26e7708ab5ed..b14589ff4842 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -279,10 +279,6 @@ jobs: needs: build runs-on: ubuntu-latest timeout-minutes: 15 - env: - # Since we build code-server we might as well run tests from the release - # since VS Code will load faster due to the bundling. - CODE_SERVER_TEST_ENTRY: "./release-packages/code-server-linux-amd64" steps: - name: Checkout repo uses: actions/checkout@v3 @@ -297,9 +293,9 @@ jobs: uses: actions/cache@v3 with: path: "**/node_modules" - key: yarn-build-${{ hashFiles('**/yarn.lock') }} + key: 1-yarn-build-${{ hashFiles('**/yarn.lock') }} restore-keys: | - yarn-build- + 1-yarn-build- - name: Download npm package uses: actions/download-artifact@v3 @@ -341,10 +337,6 @@ jobs: needs: build runs-on: ubuntu-latest timeout-minutes: 25 - env: - # Since we build code-server we might as well run tests from the release - # since VS Code will load faster due to the bundling. - CODE_SERVER_TEST_ENTRY: "./release-packages/code-server-linux-amd64" steps: - name: Checkout repo uses: actions/checkout@v3 @@ -359,9 +351,9 @@ jobs: uses: actions/cache@v3 with: path: "**/node_modules" - key: yarn-build-${{ hashFiles('**/yarn.lock') }} + key: 1-yarn-build-${{ hashFiles('**/yarn.lock') }} restore-keys: | - yarn-build- + 1-yarn-build- - name: Download npm package uses: actions/download-artifact@v3 From b78c687b8c5a1836a8e67f2827bd862d469457d0 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Mon, 26 Sep 2022 14:39:55 -0700 Subject: [PATCH 10/20] fixup: checkout submodules in e2e steps --- .github/workflows/build.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b14589ff4842..36ce2fffffb0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -282,6 +282,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v3 + with: + submodules: true - name: Install Node.js v16 uses: actions/setup-node@v3 @@ -340,6 +342,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v3 + with: + submodules: true - name: Install Node.js v16 uses: actions/setup-node@v3 From db3ca3fde73d32fa4360941828d80733a1259007 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Mon, 26 Sep 2022 15:07:10 -0700 Subject: [PATCH 11/20] fixup: try install in release dir --- .github/workflows/build.yaml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 36ce2fffffb0..265781a75082 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -282,7 +282,7 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v3 - with: + with: submodules: true - name: Install Node.js v16 @@ -307,6 +307,9 @@ jobs: - name: Decompress npm package run: tar -xzf package.tar.gz + - name: Install release package dependencies + run: cd release && yarn install + - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' run: yarn --frozen-lockfile @@ -316,11 +319,6 @@ jobs: ./test/node_modules/.bin/playwright install-deps ./test/node_modules/.bin/playwright install - - name: Copy dependencies to release (npm package) - run: | - cp -r node_modules release/ - cp -r lib/vscode/node_modules release/lib/vscode - - name: Run end-to-end tests run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 @@ -342,7 +340,7 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v3 - with: + with: submodules: true - name: Install Node.js v16 @@ -367,6 +365,9 @@ jobs: - name: Decompress npm package run: tar -xzf package.tar.gz + - name: Install release package dependencies + run: cd release && yarn install + - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' run: yarn --frozen-lockfile @@ -396,11 +397,6 @@ jobs: - name: Start Caddy run: sudo ~/.cache/caddy/caddy start --config ./ci/Caddyfile - - name: Copy dependencies to release (npm package) - run: | - cp -r node_modules release/ - cp -r lib/vscode/node_modules release/lib/vscode - - name: Run end-to-end tests run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 From 1d5d4125e4879020d4e8ad1885dbb436f8d70f20 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 09:03:57 -0700 Subject: [PATCH 12/20] fix: copy node to release package --- .github/workflows/build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 265781a75082..8c454e6f6528 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -212,6 +212,9 @@ jobs: run: yarn release if: success() + - name: Copy node binary to release + run: ln -s $(which node) ./release/lib/node + # https://github.com/actions/upload-artifact/issues/38 - name: Compress release package run: tar -czf package.tar.gz release From a8ae64bc4084be4f0e65fa42963f441c77399ab8 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 09:04:07 -0700 Subject: [PATCH 13/20] docs: add notes about test for terminal --- test/e2e/terminal.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/terminal.test.ts b/test/e2e/terminal.test.ts index df7ceeebfc2d..a8180a654da4 100644 --- a/test/e2e/terminal.test.ts +++ b/test/e2e/terminal.test.ts @@ -30,6 +30,7 @@ describe("Integrated Terminal", [], {}, () => { expect(stdout).toMatch(address) }) + // TODO@jsjoeio - add test to make sure full code-server path works test("should be able to invoke `code-server` to open a file", async ({ codeServerPage }) => { const tmpFolderPath = await tmpdir(testName) const tmpFile = path.join(tmpFolderPath, "test-file") From a04c430c63197b963b8e767b9acf7469409e5f2f Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 09:35:56 -0700 Subject: [PATCH 14/20] fixup --- .github/workflows/build.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8c454e6f6528..773432b8f460 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -212,9 +212,6 @@ jobs: run: yarn release if: success() - - name: Copy node binary to release - run: ln -s $(which node) ./release/lib/node - # https://github.com/actions/upload-artifact/issues/38 - name: Compress release package run: tar -czf package.tar.gz release @@ -322,6 +319,9 @@ jobs: ./test/node_modules/.bin/playwright install-deps ./test/node_modules/.bin/playwright install + - name: Copy node binary to release + run: ln -s $(which node) ./release/lib/node + - name: Run end-to-end tests run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 @@ -400,6 +400,9 @@ jobs: - name: Start Caddy run: sudo ~/.cache/caddy/caddy start --config ./ci/Caddyfile + - name: Copy node binary to release + run: ln -s $(which node) ./release/lib/node + - name: Run end-to-end tests run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 From d00631557ee8edf346cac7baf87d364012976bf2 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 09:54:02 -0700 Subject: [PATCH 15/20] try cp instead --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 773432b8f460..563b921fb2fb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -401,7 +401,7 @@ jobs: run: sudo ~/.cache/caddy/caddy start --config ./ci/Caddyfile - name: Copy node binary to release - run: ln -s $(which node) ./release/lib/node + run: cp -r $(which node) ./release/lib/node - name: Run end-to-end tests run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 From 5e06fbc298ec15f44590dae0436a43434d6e4ebc Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 09:55:08 -0700 Subject: [PATCH 16/20] try this --- .github/workflows/build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 563b921fb2fb..f8093ff10f24 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -401,7 +401,9 @@ jobs: run: sudo ~/.cache/caddy/caddy start --config ./ci/Caddyfile - name: Copy node binary to release - run: cp -r $(which node) ./release/lib/node + run: | + ln -s $(which node) ./release/lib/node + ln -s $(which node) ./release/node - name: Run end-to-end tests run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 From a6c4444ece11f90cabf1f7326fc2f9694a070206 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 10:19:18 -0700 Subject: [PATCH 17/20] fixup: clean it all up --- .github/workflows/build.yaml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f8093ff10f24..e00ef4d071cc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -127,7 +127,7 @@ jobs: run: yarn lint:ts build: - name: Build + name: Build code-server runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -223,6 +223,7 @@ jobs: path: ./package.tar.gz npm: + name: Publish npm package # the npm-package gets uploaded as an artifact in Build # so we need that to complete before this runs needs: build @@ -275,15 +276,13 @@ jobs: ``` test-e2e: - name: End-to-end tests + name: Run end-to-end tests needs: build runs-on: ubuntu-latest timeout-minutes: 15 steps: - name: Checkout repo uses: actions/checkout@v3 - with: - submodules: true - name: Install Node.js v16 uses: actions/setup-node@v3 @@ -295,9 +294,9 @@ jobs: uses: actions/cache@v3 with: path: "**/node_modules" - key: 1-yarn-build-${{ hashFiles('**/yarn.lock') }} + key: yarn-build-${{ hashFiles('**/yarn.lock') }} restore-keys: | - 1-yarn-build- + yarn-build- - name: Download npm package uses: actions/download-artifact@v3 @@ -312,7 +311,7 @@ jobs: - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' - run: yarn --frozen-lockfile + run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile - name: Install Playwright OS dependencies run: | @@ -320,7 +319,9 @@ jobs: ./test/node_modules/.bin/playwright install - name: Copy node binary to release - run: ln -s $(which node) ./release/lib/node + run: | + ln -s $(which node) ./release/lib/node + ln -s $(which node) ./release/node - name: Run end-to-end tests run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 @@ -336,15 +337,13 @@ jobs: run: rm -rf ./release ./test/test-results test-e2e-proxy: - name: End-to-end tests behind proxy + name: Run end-to-end tests behind proxy needs: build runs-on: ubuntu-latest timeout-minutes: 25 steps: - name: Checkout repo uses: actions/checkout@v3 - with: - submodules: true - name: Install Node.js v16 uses: actions/setup-node@v3 @@ -356,9 +355,9 @@ jobs: uses: actions/cache@v3 with: path: "**/node_modules" - key: 1-yarn-build-${{ hashFiles('**/yarn.lock') }} + key: yarn-build-${{ hashFiles('**/yarn.lock') }} restore-keys: | - 1-yarn-build- + yarn-build- - name: Download npm package uses: actions/download-artifact@v3 @@ -373,7 +372,7 @@ jobs: - name: Install dependencies if: steps.cache-node-modules.outputs.cache-hit != 'true' - run: yarn --frozen-lockfile + run: SKIP_SUBMODULE_DEPS=1 yarn --frozen-lockfile - name: Install Playwright OS dependencies run: | From dc4f2d02a7c86f2ca97ac0c3ffaa5028183fbdd1 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 10:53:49 -0700 Subject: [PATCH 18/20] fixup: update names --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e00ef4d071cc..e2ab5e18671b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -276,7 +276,7 @@ jobs: ``` test-e2e: - name: Run end-to-end tests + name: Run e2e tests needs: build runs-on: ubuntu-latest timeout-minutes: 15 @@ -337,7 +337,7 @@ jobs: run: rm -rf ./release ./test/test-results test-e2e-proxy: - name: Run end-to-end tests behind proxy + name: Run e2e tests behind proxy needs: build runs-on: ubuntu-latest timeout-minutes: 25 From 219a28d98042463952c647031c1d911e43ce3798 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 10:55:39 -0700 Subject: [PATCH 19/20] fixup: add proxy back --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e2ab5e18671b..7e4d4c4decf4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -405,7 +405,7 @@ jobs: ln -s $(which node) ./release/node - name: Run end-to-end tests - run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e --global-timeout 840000 + run: CODE_SERVER_TEST_ENTRY=./release yarn test:e2e:proxy --global-timeout 840000 - name: Stop Caddy if: always() From 4ef8e6ebb7f2ef01e9ef94c6bb21e843184943cd Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Tue, 27 Sep 2022 10:58:49 -0700 Subject: [PATCH 20/20] fixup: add comment --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7e4d4c4decf4..1d72596e1d4f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -318,6 +318,7 @@ jobs: ./test/node_modules/.bin/playwright install-deps ./test/node_modules/.bin/playwright install + # TODO@jsjoeio - this is temp. We need to do an actual fix - name: Copy node binary to release run: | ln -s $(which node) ./release/lib/node @@ -399,6 +400,7 @@ jobs: - name: Start Caddy run: sudo ~/.cache/caddy/caddy start --config ./ci/Caddyfile + # TODO@jsjoeio - this is temp. We need to do an actual fix - name: Copy node binary to release run: | ln -s $(which node) ./release/lib/node