From eed8d58be50cba95166ac3b0c6a5818268502a90 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Mon, 31 Jan 2022 13:49:45 -0700 Subject: [PATCH 1/5] feat: refactor npm workflows to use download-artifact This refactors the npm workflows to use the download-artifact GitHub Action. We had problems in the past with our download_artifact custom bash function. This also fixes an issue where we weren't downloading the correct artifacts when publishing beta and dev tags to npm. --- .github/workflows/ci.yaml | 36 +++++++++++++++++++ .github/workflows/npm-beta.yaml | 29 --------------- .github/workflows/npm-brew.yaml | 3 +- .github/workflows/npm-dev.yaml | 30 ---------------- ci/steps/publish-npm.sh | 63 +++++++++++++++++++++++++-------- 5 files changed, 85 insertions(+), 76 deletions(-) delete mode 100644 .github/workflows/npm-beta.yaml delete mode 100644 .github/workflows/npm-dev.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index aed480e3c417..66866477c9f2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -177,6 +177,42 @@ jobs: name: npm-package path: ./package.tar.gz + npm: + # the npm-package gets uploaded as an artifact in Build + # so we need that to complete before this runs + needs: build + # This environment "npm" requires someone from + # coder/code-server-reviewers to approve the PR before this job runs. + environment: npm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + # NOTES@jsjoeio - trying out + - uses: actions/download-artifact@v2 + id: download + with: + name: "npm-package" + path: release-npm-package + + - name: + Set NPM_ENVIRONMENT + # The way this logic works is it checks if the GitHub event is a push to the `main` branch + # if so, we're in the staging environment (i.e. running in CI on merge into `main`) + # otherwise it's running in CI on a PR event, meaning development environment + # Source: https://kevsoft.net/2020/06/10/running-github-action-steps-and-jobs-only-on-push-to-master.html + run: | + if [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/heads/master' ]]; then + echo "NPM_ENVIRONMENT=staging" >> "$GITHUB_ENV" + else + echo "NPM_ENVIRONMENT=development" >> "$GITHUB_ENV" + fi + - name: Run ./ci/steps/publish-npm.sh + run: yarn publish:npm + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + # TODO: cache building yarn --production # possibly 2m30s of savings(?) # this requires refactoring our release scripts diff --git a/.github/workflows/npm-beta.yaml b/.github/workflows/npm-beta.yaml deleted file mode 100644 index 4ed59e4adbb7..000000000000 --- a/.github/workflows/npm-beta.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Publish on npm and tag with "beta" - -on: - # Shows the manual trigger in GitHub UI - # helpful as a back-up in case the GitHub Actions Workflow fails - workflow_dispatch: - - push: - branches: - - main - -jobs: - # NOTE: this job requires curl, jq and yarn - # All of them are included in ubuntu-latest. - npm: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Publish npm package and tag "beta" - run: yarn publish:npm - env: - ENVIRONMENT: "staging" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - NPM_TAG: "beta" - # Since this only runs on a merge into main, we can't use github.event.number - # so we instead use the word "beta" and the PR merge commit SHA - PR_NUMBER_AND_COMMIT_SHA: beta-${{ github.sha }} diff --git a/.github/workflows/npm-brew.yaml b/.github/workflows/npm-brew.yaml index c0fdcc50a187..ebd5742948b2 100644 --- a/.github/workflows/npm-brew.yaml +++ b/.github/workflows/npm-brew.yaml @@ -19,10 +19,9 @@ jobs: - name: Publish npm package and tag with "latest" run: yarn publish:npm env: - ENVIRONMENT: "production" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - NPM_TAG: "latest" + NPM_ENVIRONMENT: "production" homebrew: # The newest version of code-server needs to be available on npm when this runs diff --git a/.github/workflows/npm-dev.yaml b/.github/workflows/npm-dev.yaml deleted file mode 100644 index 4c120284a2b9..000000000000 --- a/.github/workflows/npm-dev.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Publish on npm and tag with PR number - -on: - # Shows the manual trigger in GitHub UI - # helpful as a back-up in case the GitHub Actions Workflow fails - workflow_dispatch: - - pull_request: - branches: - - main - -jobs: - # NOTE: this job requires curl, jq and yarn - # All of them are included in ubuntu-latest. - npm: - # This environment "npm" requires someone from - # coder/code-server-reviewers to approve the PR before this job runs. - environment: npm - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Run ./ci/steps/publish-npm.sh - run: yarn publish:npm - env: - ENVIRONMENT: "development" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - NPM_TAG: ${{ github.event.number }} - PR_NUMBER_AND_COMMIT_SHA: ${{ github.event.number }}-${{ github.event.pull_request.head.sha }} diff --git a/ci/steps/publish-npm.sh b/ci/steps/publish-npm.sh index a4636db5a36e..c34d330d37e8 100755 --- a/ci/steps/publish-npm.sh +++ b/ci/steps/publish-npm.sh @@ -30,8 +30,8 @@ main() { # a developer to install this version with `yarn add code-server@beta` # "production" - this means we tag with `latest` (default), allowing # a developer to install this version with `yarn add code-server@latest` - if ! is_env_var_set "ENVIRONMENT"; then - echo "ENVIRONMENT is not set. Cannot determine npm tag without ENVIRONMENT." + if ! is_env_var_set "NPM_ENVIRONMENT"; then + echo "NPM_ENVIRONMENT is not set. Cannot determine npm tag without NPM_ENVIRONMENT." exit 1 fi @@ -47,22 +47,37 @@ main() { exit 1 fi - # We need TAG to know what to publish under on npm - # Options are "latest", "beta", or "" - # See Environment comments above to know when each is used. - if ! is_env_var_set "NPM_TAG"; then - echo "NPM_TAG is not set. This is needed for tagging the npm release." + # We use this to grab the PR_NUMBER + if ! is_env_var_set "GITHUB_REF"; then + echo "GITHUB_REF is not set. Are you running this locally? We rely on values provided by GitHub." exit 1 fi - echo "using tag: $NPM_TAG" + # We use this to grab the branch name + if ! is_env_var_set "GITHUB_HEAD_REF"; then + echo "GITHUB_HEAD_REF is not set. Are you running this locally? We rely on values provided by GitHub." + exit 1 + fi + + # We use this when setting NPM_VERSION + if ! is_env_var_set "GITHUB_SHA"; then + echo "GITHUB_SHA is not set. Are you running this locally? We rely on values provided by GitHub." + exit 1 + fi # This allows us to publish to npm in CI workflows if [[ ${CI-} ]]; then echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc fi - download_artifact npm-package ./release-npm-package + # Note: if this runs on a push to main or a release workflow + # There is no BRANCH so branch will be empty which is why + # we set a default. + # Source:https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables + BRANCH="${GITHUB_HEAD_REF-main}" + + # NOTE@jsjoeio - this script assumes we have the artifact downloaded on disk + # That happens in CI as a step before we run this. # https://github.com/actions/upload-artifact/issues/38 tar -xzf release-npm-package/package.tar.gz @@ -74,22 +89,40 @@ main() { # We only need to run npm version for "development" and "staging". # This is because our release:prep script automatically bumps the version # in the package.json and we commit it as part of the release PR. - if [[ "$ENVIRONMENT" == "production" ]]; then + if [[ "$NPM_ENVIRONMENT" == "production" ]]; then NPM_VERSION="$VERSION" + # This means the npm version will be published as "stable" + # and installed when a user runs `yarn install code-server` + NPM_TAG="latest" else + COMMIT_SHA="$GITHUB_SHA" echo "Not a production environment" - echo "Found environment: $ENVIRONMENT" + echo "Found environment: $NPM_ENVIRONMENT" echo "Manually bumping npm version..." - if ! is_env_var_set "PR_NUMBER_AND_COMMIT_SHA"; then - echo "PR_NUMBER_AND_COMMIT_SHA is not set. This is needed for setting the npm version in non-production environments." - exit 1 + if [[ "$NPM_ENVIRONMENT" == "staging" ]]; then + NPM_VERSION="$VERSION-beta-$COMMIT_SHA" + # This means the npm version will be tagged with "beta" + # and installed when a user runs `yarn install code-server@beta` + NPM_TAG="beta" + fi + + if [[ "$NPM_ENVIRONMENT" == "development" ]]; then + # Source: https://github.com/actions/checkout/issues/58#issuecomment-614041550 + PR_NUMBER=$(echo "$GITHUB_REF" | awk 'BEGIN { FS = "/" } ; { print $3 }') + NPM_VERSION="$VERSION-$PR_NUMBER-$COMMIT_SHA" + # This means the npm version will be tagged with "" + # and installed when a user runs `yarn install code-server@` + NPM_TAG="$PR_NUMBER" fi + echo "using tag: $NPM_TAG" + # We modify the version in the package.json # to be the current version + the PR number + commit SHA + # or we use current version + beta + commit SHA # Example: "version": "4.0.1-4769-ad7b23cfe6ffd72914e34781ef7721b129a23040" - NPM_VERSION="$VERSION-$PR_NUMBER_AND_COMMIT_SHA" + # Example: "version": "4.0.1-beta-ad7b23cfe6ffd72914e34781ef7721b129a23040" pushd release # NOTE:@jsjoeio # I originally tried to use `yarn version` but ran into issues and abandoned it. From 21bc065c204aae1cebe7c0b5d1c6f4b3c71ffcea Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Mon, 31 Jan 2022 16:14:18 -0700 Subject: [PATCH 2/5] fixup: remove unused env var --- ci/steps/publish-npm.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ci/steps/publish-npm.sh b/ci/steps/publish-npm.sh index c34d330d37e8..f656ec217b02 100755 --- a/ci/steps/publish-npm.sh +++ b/ci/steps/publish-npm.sh @@ -53,12 +53,6 @@ main() { exit 1 fi - # We use this to grab the branch name - if ! is_env_var_set "GITHUB_HEAD_REF"; then - echo "GITHUB_HEAD_REF is not set. Are you running this locally? We rely on values provided by GitHub." - exit 1 - fi - # We use this when setting NPM_VERSION if ! is_env_var_set "GITHUB_SHA"; then echo "GITHUB_SHA is not set. Are you running this locally? We rely on values provided by GitHub." @@ -70,12 +64,6 @@ main() { echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc fi - # Note: if this runs on a push to main or a release workflow - # There is no BRANCH so branch will be empty which is why - # we set a default. - # Source:https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables - BRANCH="${GITHUB_HEAD_REF-main}" - # NOTE@jsjoeio - this script assumes we have the artifact downloaded on disk # That happens in CI as a step before we run this. # https://github.com/actions/upload-artifact/issues/38 From d2758ddc95d25f11484459617980425b6c0ca4b1 Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Thu, 3 Feb 2022 12:56:36 -0700 Subject: [PATCH 3/5] fixup! add download-artifcat to npm-brew" --- .github/workflows/ci.yaml | 1 - .github/workflows/npm-brew.yaml | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 66866477c9f2..cc04e8fe6830 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -188,7 +188,6 @@ jobs: steps: - uses: actions/checkout@v2 - # NOTES@jsjoeio - trying out - uses: actions/download-artifact@v2 id: download with: diff --git a/.github/workflows/npm-brew.yaml b/.github/workflows/npm-brew.yaml index ebd5742948b2..a515e423e1ff 100644 --- a/.github/workflows/npm-brew.yaml +++ b/.github/workflows/npm-brew.yaml @@ -16,6 +16,12 @@ jobs: steps: - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + id: download + with: + name: "npm-package" + path: release-npm-package + - name: Publish npm package and tag with "latest" run: yarn publish:npm env: From 955145d97e3dd7e0d4889623874c40797779eebf Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Thu, 3 Feb 2022 12:57:31 -0700 Subject: [PATCH 4/5] fixup! remove unnecessary code comment --- .github/workflows/ci.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cc04e8fe6830..7a33f8af93e8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -194,12 +194,7 @@ jobs: name: "npm-package" path: release-npm-package - - name: - Set NPM_ENVIRONMENT - # The way this logic works is it checks if the GitHub event is a push to the `main` branch - # if so, we're in the staging environment (i.e. running in CI on merge into `main`) - # otherwise it's running in CI on a PR event, meaning development environment - # Source: https://kevsoft.net/2020/06/10/running-github-action-steps-and-jobs-only-on-push-to-master.html + - name: Set NPM_ENVIRONMENT run: | if [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/heads/master' ]]; then echo "NPM_ENVIRONMENT=staging" >> "$GITHUB_ENV" From 0dc88c8da117fb24cfd88ff6f8d32fbe357c934d Mon Sep 17 00:00:00 2001 From: Joe Previte Date: Thu, 3 Feb 2022 13:12:09 -0700 Subject: [PATCH 5/5] fixup! move NPM_ENVIRONMENT logic to script --- .github/workflows/ci.yaml | 11 ++++------- ci/steps/publish-npm.sh | 41 ++++++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7a33f8af93e8..fc96569973a0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -194,18 +194,15 @@ jobs: name: "npm-package" path: release-npm-package - - name: Set NPM_ENVIRONMENT - run: | - if [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/heads/master' ]]; then - echo "NPM_ENVIRONMENT=staging" >> "$GITHUB_ENV" - else - echo "NPM_ENVIRONMENT=development" >> "$GITHUB_ENV" - fi - name: Run ./ci/steps/publish-npm.sh run: yarn publish:npm env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + # NOTE@jsjoeio + # NPM_ENVIRONMENT intentionally not set here. + # Instead, itis determined in publish-npm.sh script + # using GITHUB environment variables # TODO: cache building yarn --production # possibly 2m30s of savings(?) diff --git a/ci/steps/publish-npm.sh b/ci/steps/publish-npm.sh index f656ec217b02..9042a7ccda36 100755 --- a/ci/steps/publish-npm.sh +++ b/ci/steps/publish-npm.sh @@ -21,20 +21,6 @@ main() { exit 1 fi - ## Environment - # This string is used to determine how we should tag the npm release. - # Environment can be one of three choices: - # "development" - this means we tag with the PR number, allowing - # a developer to install this version with `yarn add code-server@` - # "staging" - this means we tag with `beta`, allowing - # a developer to install this version with `yarn add code-server@beta` - # "production" - this means we tag with `latest` (default), allowing - # a developer to install this version with `yarn add code-server@latest` - if ! is_env_var_set "NPM_ENVIRONMENT"; then - echo "NPM_ENVIRONMENT is not set. Cannot determine npm tag without NPM_ENVIRONMENT." - exit 1 - fi - ## Publishing Information # All the variables below are used to determine how we should publish # the npm package. We also use this information for bumping the version. @@ -59,11 +45,38 @@ main() { exit 1 fi + # We use this to determine the NPM_ENVIRONMENT + if ! is_env_var_set "GITHUB_EVENT_NAME"; then + echo "GITHUB_EVENT_NAME is not set. Are you running this locally? We rely on values provided by GitHub." + exit 1 + fi + # This allows us to publish to npm in CI workflows if [[ ${CI-} ]]; then echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc fi + ## Environment + # This string is used to determine how we should tag the npm release. + # Environment can be one of three choices: + # "development" - this means we tag with the PR number, allowing + # a developer to install this version with `yarn add code-server@` + # "staging" - this means we tag with `beta`, allowing + # a developer to install this version with `yarn add code-server@beta` + # "production" - this means we tag with `latest` (default), allowing + # a developer to install this version with `yarn add code-server@latest` + if ! is_env_var_set "NPM_ENVIRONMENT"; then + echo "NPM_ENVIRONMENT is not set. Determining in script based on GITHUB environment variables." + + if [[ "$GITHUB_EVENT_NAME" == 'push' && "$GITHUB_REF" == 'refs/heads/main' ]]; then + NPM_ENVIRONMENT="staging" + else + NPM_ENVIRONMENT="development" + fi + + echo "Using npm environment: $NPM_ENVIRONMENT" + fi + # NOTE@jsjoeio - this script assumes we have the artifact downloaded on disk # That happens in CI as a step before we run this. # https://github.com/actions/upload-artifact/issues/38