From a25352a27fd1fb18f44386e8a54f12ed1939994b Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 8 May 2023 14:44:20 +0200 Subject: [PATCH 1/9] chore(ci): build changelog on trunk changes --- .github/workflows/build_changelog.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_changelog.yml b/.github/workflows/build_changelog.yml index ebc978022bc..0964e568247 100644 --- a/.github/workflows/build_changelog.yml +++ b/.github/workflows/build_changelog.yml @@ -3,18 +3,13 @@ name: Build changelog on: workflow_dispatch: - schedule: - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - # │ │ │ │ │ - # │ │ │ │ │ - # │ │ │ │ │ - # * * * * * - - cron: '0 8 * * *' + push: + branches: + - develop jobs: changelog: + permissions: + contents: write + pull-requests: write uses: ./.github/workflows/reusable_publish_changelog.yml From c8a19de4bacb6cdc5a08568c76124843e4a1621d Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 8 May 2023 15:01:28 +0200 Subject: [PATCH 2/9] chore: reuse PR_TITLE as commit message, making it easier to use Signed-off-by: heitorlessa --- .github/scripts/create_pr_for_staged_changes.sh | 5 ++--- .github/workflows/reusable_publish_changelog.yml | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/scripts/create_pr_for_staged_changes.sh b/.github/scripts/create_pr_for_staged_changes.sh index a35d45cc9e9..dd8ef7e6d59 100644 --- a/.github/scripts/create_pr_for_staged_changes.sh +++ b/.github/scripts/create_pr_for_staged_changes.sh @@ -10,7 +10,7 @@ set -uxo pipefail # enable debugging, prevent accessing unset env vars, prevent #usage :bash create_pr_for_staged_changes.sh {git_staged_files_or_directories_separated_by_space} #notes :Meant to use in GitHub Actions only. Temporary branch will be named $TEMP_BRANCH_PREFIX-$GITHUB_RUN_ID #os_version :Ubuntu 22.04.2 LTS -#required_env_vars :COMMIT_MSG, PR_TITLE, TEMP_BRANCH_PREFIX, GH_TOKEN, GITHUB_RUN_ID, GITHUB_SERVER_URL, GITHUB_REPOSITORY +#required_env_vars :PR_TITLE, TEMP_BRANCH_PREFIX, GH_TOKEN, GITHUB_RUN_ID, GITHUB_SERVER_URL, GITHUB_REPOSITORY #============================================================================== PR_BODY="This is an automated PR created from the following workflow" @@ -37,7 +37,6 @@ function has_required_config() { debug "Do we have required environment variables?" test -z "${TEMP_BRANCH_PREFIX}" && raise_validation_error "TEMP_BRANCH_PREFIX env must be set to create a PR" test -z "${GH_TOKEN}" && raise_validation_error "GH_TOKEN env must be set for GitHub CLI" - test -z "${COMMIT_MSG}" && raise_validation_error "COMMIT_MSG env must be set" test -z "${PR_TITLE}" && raise_validation_error "PR_TITLE env must be set" test -z "${GITHUB_RUN_ID}" && raise_validation_error "GITHUB_RUN_ID env must be set to trace Workflow Run ID back to PR" test -z "${GITHUB_SERVER_URL}" && raise_validation_error "GITHUB_SERVER_URL env must be set to trace Workflow Run ID back to PR" @@ -67,7 +66,7 @@ function create_temporary_branch_with_changes() { debug "Committing staged files: $*" git add "$@" - git commit -m "${COMMIT_MSG}" + git commit -m "${PR_TITLE}" debug "Creating branch remotely" git push origin "${TEMP_BRANCH}" diff --git a/.github/workflows/reusable_publish_changelog.yml b/.github/workflows/reusable_publish_changelog.yml index 4294dda4a94..e8d3e540e43 100644 --- a/.github/workflows/reusable_publish_changelog.yml +++ b/.github/workflows/reusable_publish_changelog.yml @@ -33,8 +33,7 @@ jobs: - name: Create PR run: bash .github/scripts/create_pr_for_staged_changes.sh CHANGELOG.md env: - COMMIT_MSG: "chore(ci): update changelog with latest changes" - PR_TITLE: "chore(ci): changelog rebuild" + PR_TITLE: "chore(ci): changelog rebuild with latest changes" TEMP_BRANCH_PREFIX: "ci-changelog" GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Cleanup orphaned branch From 7710d6542f4bdb695bf9f53d73930eee4a8cca31 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 8 May 2023 15:35:15 +0200 Subject: [PATCH 3/9] chore(ci): create layer docs pr, not push Signed-off-by: heitorlessa --- .github/workflows/publish_v2_layer.yml | 39 ++++++++++++++++--- .github/workflows/release.yml | 1 + .../reusable_deploy_v2_layer_stack.yml | 15 +++++-- .../reusable_update_v2_layer_arn_docs.yml | 24 ++++++++---- 4 files changed, 63 insertions(+), 16 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 7820678e813..c356d05ff58 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -1,10 +1,5 @@ name: Deploy v2 layer to all regions -permissions: - id-token: write - contents: write - pages: write - on: workflow_dispatch: inputs: @@ -32,6 +27,9 @@ jobs: build-layer: permissions: contents: read + id-token: none + pages: none + pull-requests: none runs-on: aws-lambda-powertools_ubuntu-latest_8-core defaults: run: @@ -87,6 +85,12 @@ jobs: beta: needs: build-layer + # lower privilege propagated from parent workflow (release.yml) + permissions: + id-token: write + contents: write + pull-requests: write + pages: none uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml secrets: inherit with: @@ -97,6 +101,12 @@ jobs: prod: needs: beta + # lower privilege propagated from parent workflow (release.yml) + permissions: + id-token: write + contents: write + pull-requests: write + pages: none uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml secrets: inherit with: @@ -107,6 +117,12 @@ jobs: sar-beta: needs: build-layer + permissions: + # lower privilege propagated from parent workflow (release.yml) + id-token: write + contents: read + pull-requests: none + pages: none uses: ./.github/workflows/reusable_deploy_v2_sar.yml secrets: inherit with: @@ -117,6 +133,12 @@ jobs: sar-prod: needs: [build-layer, sar-beta] + permissions: + # lower privilege propagated from parent workflow (release.yml) + id-token: write + contents: read + pull-requests: none + pages: none uses: ./.github/workflows/reusable_deploy_v2_sar.yml secrets: inherit with: @@ -128,7 +150,11 @@ jobs: prepare_docs_alias: runs-on: ubuntu-latest permissions: + # lower privilege propagated from parent workflow (release.yml) contents: read + pages: none + id-token: none + pull-requests: none outputs: DOCS_ALIAS: ${{ steps.set-alias.outputs.DOCS_ALIAS }} steps: @@ -144,8 +170,11 @@ jobs: release-docs: needs: [prod, prepare_docs_alias] permissions: + # lower privilege propagated from parent workflow (release.yml) contents: write pages: write + id-token: write + pull-requests: none uses: ./.github/workflows/reusable_publish_docs.yml with: version: ${{ inputs.latest_published_version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38aadc4d873..4f93ceab1db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -158,6 +158,7 @@ jobs: id-token: write contents: write pages: write + pull-requests: write uses: ./.github/workflows/publish_v2_layer.yml with: latest_published_version: ${{ needs.build.outputs.RELEASE_VERSION }} diff --git a/.github/workflows/reusable_deploy_v2_layer_stack.yml b/.github/workflows/reusable_deploy_v2_layer_stack.yml index 912849c2e2c..cf0ce8230eb 100644 --- a/.github/workflows/reusable_deploy_v2_layer_stack.yml +++ b/.github/workflows/reusable_deploy_v2_layer_stack.yml @@ -1,8 +1,5 @@ name: Deploy CDK Layer v2 stack -permissions: - id-token: write - contents: write on: workflow_call: @@ -28,6 +25,12 @@ jobs: deploy-cdk-stack: runs-on: ubuntu-latest environment: ${{ inputs.environment }} + # lower privilege propagated from parent workflow (publish_v2_layer.yml) + permissions: + id-token: write + pull-requests: none + contents: read + pages: none defaults: run: working-directory: ./layer @@ -153,6 +156,12 @@ jobs: update_v2_layer_arn_docs: needs: deploy-cdk-stack if: ${{ inputs.stage == 'PROD' }} + # lower privilege propagated from parent workflow (publish_v2_layer.yml) + permissions: + contents: write + pull-requests: write + pages: none + id-token: none uses: ./.github/workflows/reusable_update_v2_layer_arn_docs.yml with: latest_published_version: ${{ inputs.latest_published_version }} diff --git a/.github/workflows/reusable_update_v2_layer_arn_docs.yml b/.github/workflows/reusable_update_v2_layer_arn_docs.yml index 5fbf6814dcd..68ad6efb9ce 100644 --- a/.github/workflows/reusable_update_v2_layer_arn_docs.yml +++ b/.github/workflows/reusable_update_v2_layer_arn_docs.yml @@ -9,7 +9,11 @@ on: required: true permissions: + # lower privilege propagated from parent workflow (reusable_deploy_v2_layer_stack.yml) contents: write + pull-requests: write + pages: none + id-token: none env: BRANCH: develop @@ -42,11 +46,15 @@ jobs: run: | ls -la cdk-layer-stack/ ./layer/scripts/update_layer_arn.sh cdk-layer-stack - - name: Update documentation in trunk - run: | - HAS_CHANGE=$(git status --porcelain) - test -z "${HAS_CHANGE}" && echo "Nothing to update" && exit 0 - git add docs/index.md examples - git commit -m "chore: update v2 layer ARN on documentation" - git pull origin "${BRANCH}" # prevents concurrent branch update failing push - git push origin HEAD:refs/heads/"${BRANCH}" + - name: Create PR + run: bash .github/scripts/create_pr_for_staged_changes.sh docs/index.md examples + env: + PR_TITLE: "docs(layers): update v2 layer ARN on documentation" + TEMP_BRANCH_PREFIX: "ci-docs" + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Cleanup orphaned branch + if: failure() + run: git push origin --delete "${TEMP_BRANCH_PREFIX}-${GITHUB_RUN_ID}" || echo "Must have failed before creating temporary branch; no cleanup needed." + env: + TEMP_BRANCH_PREFIX: "ci-docs" + GITHUB_RUN_ID: ${{ github.run_id }} From 2d0a0200714dbcc1e0d8790c20350f0e09c66748 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 8 May 2023 15:55:37 +0200 Subject: [PATCH 4/9] chore(ci): create PR to bump version; drop now redundant changelog job --- .github/workflows/release.yml | 40 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4f93ceab1db..63a364febda 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,16 +105,6 @@ jobs: # and also future-proof for when we switch to protected branch and update via PR key: ${{ runner.os }}-${{ env.RELEASE_VERSION }}-${{ hashFiles('**/poetry.lock') }} - - name: Update version in trunk - if: steps.versioning.outcome == 'success' - run: | - HAS_CHANGE=$(git status --porcelain) - test -z "${HAS_CHANGE}" && echo "Nothing to update" && exit 0 - git add pyproject.toml - git commit -m "bump version to ${RELEASE_VERSION}" --no-verify - git pull origin "${BRANCH}" # prevents concurrent branch update failing push - git push origin HEAD:refs/heads/"${BRANCH}" - release: needs: build environment: release @@ -143,12 +133,6 @@ jobs: # with: # repository-url: https://test.pypi.org/legacy/ - changelog: - needs: release - permissions: - contents: write - uses: ./.github/workflows/reusable_publish_changelog.yml - # NOTE: Watch out for the depth limit of 4 nested workflow_calls. # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack -> reusable_update_v2_layer_arn_docs publish_layer: @@ -183,3 +167,27 @@ jobs: script: | const post_release = require('.github/scripts/post_release.js') await post_release({github, context, core}) + + - name: Git client setup and refresh tip + run: | + git config user.name "Release bot" + git config user.email "aws-devax-open-source@amazon.com" + git config pull.rebase true + git config remote.origin.url >&- + git pull origin "${BRANCH}" + + - name: Bump package version + id: versioning + run: poetry version "${RELEASE_VERSION}" + - name: Create PR + run: bash .github/scripts/create_pr_for_staged_changes.sh pyproject.toml + env: + PR_TITLE: "chore(ci): bump version to ${{ env.RELEASE_VERSION }}" + TEMP_BRANCH_PREFIX: "ci-bump" + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Cleanup orphaned branch + if: failure() + run: git push origin --delete "${TEMP_BRANCH_PREFIX}-${GITHUB_RUN_ID}" || echo "Must have failed before creating temporary branch; no cleanup needed." + env: + TEMP_BRANCH_PREFIX: "ci-bump" + GITHUB_RUN_ID: ${{ github.run_id }} From bc6b9b97273e3af91429ea4baf70448b4521dd6b Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 8 May 2023 16:01:06 +0200 Subject: [PATCH 5/9] chore(ci): breakdown post_release to increase security --- .github/workflows/release.yml | 36 ++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 63a364febda..55c2b35485a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -148,26 +148,16 @@ jobs: latest_published_version: ${{ needs.build.outputs.RELEASE_VERSION }} pre_release: ${{ inputs.pre_release }} - post_release: - needs: [build, release, publish_layer] + bump_version: + needs: [build, release] permissions: - contents: read - issues: write - discussions: write + contents: write pull-requests: write runs-on: ubuntu-latest env: RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - - name: Close issues related to this release - uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const post_release = require('.github/scripts/post_release.js') - await post_release({github, context, core}) - - name: Git client setup and refresh tip run: | git config user.name "Release bot" @@ -191,3 +181,23 @@ jobs: env: TEMP_BRANCH_PREFIX: "ci-bump" GITHUB_RUN_ID: ${{ github.run_id }} + + notify_version: + needs: [build, release, publish_layer] + permissions: + contents: read + issues: write + discussions: write + pull-requests: write + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + - name: Close issues related to this release + uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const post_release = require('.github/scripts/post_release.js') + await post_release({github, context, core}) From a9d9c3e45239eb4cc5017018a580d2190f72e9a3 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 10 May 2023 18:07:17 +0200 Subject: [PATCH 6/9] chore: sync with latest working script --- .../scripts/create_pr_for_staged_changes.sh | 104 +++++++++++------- .github/workflows/publish_v2_layer.yml | 4 + .github/workflows/release.yml | 4 +- 3 files changed, 73 insertions(+), 39 deletions(-) diff --git a/.github/scripts/create_pr_for_staged_changes.sh b/.github/scripts/create_pr_for_staged_changes.sh index dd8ef7e6d59..08a8b70aebe 100644 --- a/.github/scripts/create_pr_for_staged_changes.sh +++ b/.github/scripts/create_pr_for_staged_changes.sh @@ -1,103 +1,132 @@ #!/bin/bash -set -uxo pipefail # enable debugging, prevent accessing unset env vars, prevent masking pipeline errors to the next command +set -uo pipefail # prevent accessing unset env vars, prevent masking pipeline errors to the next command #docs #title :create_pr_for_staged_changes.sh -#description :This script will create a PR for staged changes and detect and close duplicate PRs. +#description :This script will create a PR for staged changes, detect and close duplicate PRs. #author :@heitorlessa #date :May 8th 2023 #version :0.1 #usage :bash create_pr_for_staged_changes.sh {git_staged_files_or_directories_separated_by_space} #notes :Meant to use in GitHub Actions only. Temporary branch will be named $TEMP_BRANCH_PREFIX-$GITHUB_RUN_ID #os_version :Ubuntu 22.04.2 LTS -#required_env_vars :PR_TITLE, TEMP_BRANCH_PREFIX, GH_TOKEN, GITHUB_RUN_ID, GITHUB_SERVER_URL, GITHUB_REPOSITORY +#required_env_vars :PR_TITLE, TEMP_BRANCH_PREFIX, GH_TOKEN #============================================================================== -PR_BODY="This is an automated PR created from the following workflow" -FILENAME=".github/scripts/$(basename "$0")" -readonly PR_BODY -readonly FILENAME - # Sets GitHub Action with error message to ease troubleshooting -function raise_validation_error() { +function error() { echo "::error file=${FILENAME}::$1" exit 1 } function debug() { - echo "::debug::$1" + TIMESTAMP=$(date -u "+%FT%TZ") # 2023-05-10T07:53:59Z + echo ""${TIMESTAMP}" - $1" } function notice() { echo "::notice file=${FILENAME}::$1" } +function start_span() { + echo "::group::$1" +} + +function end_span() { + echo "::endgroup::" +} + function has_required_config() { + start_span "Validating required config" + test -z "${TEMP_BRANCH_PREFIX}" && error "TEMP_BRANCH_PREFIX env must be set to create a PR" + test -z "${PR_TITLE}" && error "PR_TITLE env must be set" + test -z "${GH_TOKEN}" && error "GH_TOKEN env must be set for GitHub CLI" + # Default GitHub Actions Env Vars: https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables - debug "Do we have required environment variables?" - test -z "${TEMP_BRANCH_PREFIX}" && raise_validation_error "TEMP_BRANCH_PREFIX env must be set to create a PR" - test -z "${GH_TOKEN}" && raise_validation_error "GH_TOKEN env must be set for GitHub CLI" - test -z "${PR_TITLE}" && raise_validation_error "PR_TITLE env must be set" - test -z "${GITHUB_RUN_ID}" && raise_validation_error "GITHUB_RUN_ID env must be set to trace Workflow Run ID back to PR" - test -z "${GITHUB_SERVER_URL}" && raise_validation_error "GITHUB_SERVER_URL env must be set to trace Workflow Run ID back to PR" - test -z "${GITHUB_REPOSITORY}" && raise_validation_error "GITHUB_REPOSITORY env must be set to trace Workflow Run ID back to PR" + debug "Are we running in GitHub Action environment?" + test -z "${GITHUB_RUN_ID}" && error "GITHUB_RUN_ID env must be set to trace Workflow Run ID back to PR" + test -z "${GITHUB_SERVER_URL}" && error "GITHUB_SERVER_URL env must be set to trace Workflow Run ID back to PR" + test -z "${GITHUB_REPOSITORY}" && error "GITHUB_REPOSITORY env must be set to trace Workflow Run ID back to PR" + debug "Config validated successfully!" set_environment_variables + end_span } function set_environment_variables() { - WORKFLOW_URL="${GITHUB_SERVER_URL}"/"${GITHUB_REPOSITORY}"/actions/runs/"${GITHUB_RUN_ID}" # e.g., heitorlessa/aws-lambda-powertools-test/actions/runs/4913570678 - TEMP_BRANCH="${TEMP_BRANCH_PREFIX}"-"${GITHUB_RUN_ID}" # e.g., ci-changelog-4894658712 - - export readonly WORKFLOW_URL - export readonly TEMP_BRANCH + start_span "Setting environment variables" + export readonly WORKFLOW_URL="${GITHUB_SERVER_URL}"/"${GITHUB_REPOSITORY}"/actions/runs/"${GITHUB_RUN_ID}" # e.g., heitorlessa/aws-lambda-powertools-test/actions/runs/4913570678 + export readonly TEMP_BRANCH="${TEMP_BRANCH_PREFIX}"-"${GITHUB_RUN_ID}" # e.g., ci-changelog-4894658712 + export readonly BASE_BRANCH="${BASE_BRANCH:-develop}" # e.g., main, defaults to develop if missing + export readonly PR_BODY="This is an automated PR created from the following workflow" + export readonly FILENAME=".github/scripts/$(basename "$0")" + export readonly NO_DUPLICATES_MESSAGE="No duplicated PRs found" + end_span } function has_anything_changed() { - debug "Is there an update to the source code?" + start_span "Validating git staged files" HAS_ANY_SOURCE_CODE_CHANGED="$(git status --porcelain)" - test -z "${HAS_ANY_SOURCE_CODE_CHANGED}" && echo "Nothing to update" && exit 0 + test -z "${HAS_ANY_SOURCE_CODE_CHANGED}" && debug "Nothing to update; exitting early" && exit 0 + end_span } function create_temporary_branch_with_changes() { - debug "Creating branch ${TEMP_BRANCH}" + start_span "Creating temporary branch: "${TEMP_BRANCH}"" git checkout -b "${TEMP_BRANCH}" debug "Committing staged files: $*" - git add "$@" + echo "$@" | xargs -n1 git add || error "Failed to add staged changes: "$@"" git commit -m "${PR_TITLE}" - debug "Creating branch remotely" git push origin "${TEMP_BRANCH}" + end_span } function create_pr() { - debug "Creating PR against ${BRANCH} branch" - NEW_PR_URL=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}: ${WORKFLOW_URL}" --base "${BRANCH}") # e.g, https://github.com/awslabs/aws-lambda-powertools/pull/13 + start_span "Creating PR against ${TEMP_BRANCH} branch" + NEW_PR_URL=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}: ${WORKFLOW_URL}" --base "${BASE_BRANCH}" || error "Failed to create PR") # e.g, https://github.com/awslabs/aws-lambda-powertools/pull/13 # greedy remove any string until the last URL path, including the last '/'. https://opensource.com/article/17/6/bash-parameter-expansion + debug "Extracing PR Number from PR URL: "${NEW_PR_URL}"" NEW_PR_ID="${NEW_PR_URL##*/}" # 13 export NEW_PR_URL export NEW_PR_ID + end_span } function close_duplicate_prs() { - debug "Do we have any duplicate PRs?" + start_span "Searching for duplicate PRs" DUPLICATE_PRS=$(gh pr list --search "${PR_TITLE}" --json number --jq ".[] | select(.number != ${NEW_PR_ID}) | .number") # e.g, 13\n14 - debug "Closing duplicated PRs if any" - echo "${DUPLICATE_PRS}" | xargs -L1 gh pr close --delete-branch --comment "Superseded by #${NEW_PR_ID}" + if [ -z "${DUPLICATE_PRS}" ]; then + debug "No duplicate PRs found" + DUPLICATE_PRS="${NO_DUPLICATES_MESSAGE}" + else + debug "Closing duplicated PRs: "${DUPLICATE_PRS}"" + echo "${DUPLICATE_PRS}" | xargs -L1 gh pr close --delete-branch --comment "Superseded by #${NEW_PR_ID}" + fi + export readonly DUPLICATE_PRS + end_span +} + +function report_job_output() { + start_span "Updating job outputs" + echo "pull_request_id="${NEW_PR_ID}"" >>"$GITHUB_OUTPUT" + echo temp_branch="${TEMP_BRANCH}" >>"$GITHUB_OUTPUT" + end_span } function report_summary() { - debug "Creating job summary" - echo "### Pull request created successfully :rocket: #${NEW_PR_URL}

Closed duplicated PRs (if any): ${DUPLICATE_PRS}" >>"$GITHUB_STEP_SUMMARY" + start_span "Creating job summary" + echo "### Pull request created successfully :rocket: ${NEW_PR_URL}

Closed duplicated PRs: ${DUPLICATE_PRS}" >>"$GITHUB_STEP_SUMMARY" - notice "PR_URL is ${NEW_PR_URL}" - notice "PR_BRANCH is ${TEMP_BRANCH}" - notice "PR_DUPLICATES are ${DUPLICATE_PRS}" + notice "PR_URL is: ${NEW_PR_URL}" + notice "PR_BRANCH is: ${TEMP_BRANCH}" + notice "PR_DUPLICATES are: ${DUPLICATE_PRS}" + end_span } function main() { @@ -109,6 +138,7 @@ function main() { create_pr close_duplicate_prs + report_job_output report_summary } diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index c356d05ff58..4bcf5e4f034 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -167,6 +167,10 @@ jobs: fi echo DOCS_ALIAS="$DOCS_ALIAS" >> "$GITHUB_OUTPUT" + # TODO: Versioned docs won't have Layer releases on them + # since there will be a PR with them + # so we either need to rethink this entirely + # OR pause, merge PR and then approve this step release-docs: needs: [prod, prepare_docs_alias] permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 55c2b35485a..a213fbacdc4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ name: Release # 2. Bump package version, build release artifact, and generate latest Changelog # 3. Publish package to PyPi prod repository using cached artifact # 4. Kick off Layers pipeline to compile and publish latest version -# 5. Updates documentation to use the latest Layer ARN for all commercial regions +# 5. Create PR to updates documentation to use the latest Layer ARN for all commercial regions # 6. Builds a new user guide and API docs with release version; update /latest pointing to newly released version # 7. Close all issues labeled "pending-release" and notify customers about the release # @@ -51,7 +51,7 @@ jobs: build: runs-on: aws-lambda-powertools_ubuntu-latest_4-core permissions: - contents: write + contents: read outputs: RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }} env: From 5b13b4be27076b657bfa166b12d62538cd0ad8e5 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 10 May 2023 18:32:08 +0200 Subject: [PATCH 7/9] chore: use new custom action to create PR instead --- .github/actions/create-pr/action.yml | 56 +++++++++++++++++++ .../create_pr_for_staged_changes.sh | 2 +- .../workflows/reusable_publish_changelog.yml | 27 ++------- 3 files changed, 63 insertions(+), 22 deletions(-) create mode 100644 .github/actions/create-pr/action.yml rename .github/{scripts => actions/create-pr}/create_pr_for_staged_changes.sh (98%) mode change 100644 => 100755 diff --git a/.github/actions/create-pr/action.yml b/.github/actions/create-pr/action.yml new file mode 100644 index 00000000000..f3519273f25 --- /dev/null +++ b/.github/actions/create-pr/action.yml @@ -0,0 +1,56 @@ +name: "Create PR custom action" +description: "Create a PR and a temporary branch, close duplicates" + +# This custom action + +inputs: + files: + description: "Files to add" + required: true + temp_branch_prefix: + description: "Prefix for temporary git branch to be created, e.g, ci-docs" + required: true + pull_request_title: + description: "Pull Request title to use" + required: true + github_token: + description: "GitHub token for GitHub CLI" + required: true +outputs: + pull_request_id: + description: "Pull request ID created" + value: ${{ steps.create-pr.outputs.pull_request_id }} + temp_branch: + description: "Temporary branch created with staged changed" + value: ${{ steps.create-pr.outputs.temp_branch }} +runs: + using: "composite" + steps: + - id: adjust-path + run: echo "${{ github.action_path }}" >> $GITHUB_PATH + shell: bash + - id: setup-git + name: Git client setup and refresh tip + run: | + git config user.name "Powertools bot" + git config user.email "aws-lambda-powertools-feedback@amazon.com" + git config pull.rebase true + git config remote.origin.url >&- + shell: bash + - id: create-pr + working-directory: ${{ env.GITHUB_WORKSPACE }} + run: create_pr_for_staged_changes.sh "${FILES}" + env: + FILES: ${{ inputs.files }} + TEMP_BRANCH_PREFIX: ${{ inputs.temp_branch_prefix }} + GH_TOKEN: ${{ inputs.github_token }} + PR_TITLE: ${{ inputs.pull_request_title }} + shell: bash + - id: cleanup + name: Cleanup orphaned branch + if: failure() + run: git push origin --delete "${TEMP_BRANCH_PREFIX}-${GITHUB_RUN_ID}" || echo "Must have failed before creating temporary branch; no cleanup needed." + env: + TEMP_BRANCH_PREFIX: ${{ inputs.temp_branch_prefix }} + GITHUB_RUN_ID: ${{ github.run_id }} + shell: bash diff --git a/.github/scripts/create_pr_for_staged_changes.sh b/.github/actions/create-pr/create_pr_for_staged_changes.sh old mode 100644 new mode 100755 similarity index 98% rename from .github/scripts/create_pr_for_staged_changes.sh rename to .github/actions/create-pr/create_pr_for_staged_changes.sh index 08a8b70aebe..2f32ab24342 --- a/.github/scripts/create_pr_for_staged_changes.sh +++ b/.github/actions/create-pr/create_pr_for_staged_changes.sh @@ -114,7 +114,7 @@ function close_duplicate_prs() { function report_job_output() { start_span "Updating job outputs" - echo "pull_request_id="${NEW_PR_ID}"" >>"$GITHUB_OUTPUT" + echo pull_request_id="${NEW_PR_ID}" >>"$GITHUB_OUTPUT" echo temp_branch="${TEMP_BRANCH}" >>"$GITHUB_OUTPUT" end_span } diff --git a/.github/workflows/reusable_publish_changelog.yml b/.github/workflows/reusable_publish_changelog.yml index e8d3e540e43..c6adfe3bc54 100644 --- a/.github/workflows/reusable_publish_changelog.yml +++ b/.github/workflows/reusable_publish_changelog.yml @@ -3,9 +3,6 @@ name: Build and publish latest changelog on: workflow_call: -env: - BRANCH: develop - jobs: publish_changelog: # Force Github action to run only a single job at a time (based on the group name) @@ -21,24 +18,12 @@ jobs: uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 with: fetch-depth: 0 - - name: Git client setup and refresh tip - run: | - git config user.name "Release bot" - git config user.email "aws-devax-open-source@amazon.com" - git config pull.rebase true - git config remote.origin.url >&- || git remote add origin https://github.com/"${origin}" # Git Detached mode (release notes) doesn't have origin - git pull origin "${BRANCH}" - name: "Generate latest changelog" run: make changelog - name: Create PR - run: bash .github/scripts/create_pr_for_staged_changes.sh CHANGELOG.md - env: - PR_TITLE: "chore(ci): changelog rebuild with latest changes" - TEMP_BRANCH_PREFIX: "ci-changelog" - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cleanup orphaned branch - if: failure() - run: git push origin --delete "${TEMP_BRANCH_PREFIX}-${GITHUB_RUN_ID}" || echo "Must have failed before creating temporary branch; no cleanup needed." - env: - TEMP_BRANCH_PREFIX: "ci-changelog" - GITHUB_RUN_ID: ${{ github.run_id }} + uses: ./.github/actions/create-pr + with: + files: "CHANGELOG.md" + temp_branch_prefix: "ci-changelog" + pull_request_title: "chore(ci): changelog rebuild" + github_token: ${{ secrets.GITHUB_TOKEN }} From 514ea313a53a8b8f7727f22bfd82ee65a9f41389 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 10 May 2023 19:21:13 +0200 Subject: [PATCH 8/9] chore: extend publishing docs to accept git ref summary: extend publish_docs to support git reference (temp branch id) make reusable_update_v2_layer_arn_docs redundant lower permissions --- .github/workflows/publish_v2_layer.yml | 11 ++-- .../reusable_deploy_v2_layer_stack.yml | 49 +++++++++++++-- .github/workflows/reusable_publish_docs.yml | 9 ++- .../reusable_update_v2_layer_arn_docs.yml | 60 ------------------- 4 files changed, 57 insertions(+), 72 deletions(-) delete mode 100644 .github/workflows/reusable_update_v2_layer_arn_docs.yml diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 4bcf5e4f034..4212ed17a6c 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -90,7 +90,7 @@ jobs: id-token: write contents: write pull-requests: write - pages: none + pages: write uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml secrets: inherit with: @@ -106,7 +106,7 @@ jobs: id-token: write contents: write pull-requests: write - pages: none + pages: write uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml secrets: inherit with: @@ -167,11 +167,8 @@ jobs: fi echo DOCS_ALIAS="$DOCS_ALIAS" >> "$GITHUB_OUTPUT" - # TODO: Versioned docs won't have Layer releases on them - # since there will be a PR with them - # so we either need to rethink this entirely - # OR pause, merge PR and then approve this step - release-docs: + # Update the latest documentation version + release_docs: needs: [prod, prepare_docs_alias] permissions: # lower privilege propagated from parent workflow (release.yml) diff --git a/.github/workflows/reusable_deploy_v2_layer_stack.yml b/.github/workflows/reusable_deploy_v2_layer_stack.yml index cf0ce8230eb..1a848731c7b 100644 --- a/.github/workflows/reusable_deploy_v2_layer_stack.yml +++ b/.github/workflows/reusable_deploy_v2_layer_stack.yml @@ -153,15 +153,56 @@ jobs: - name: CDK Deploy Canary run: npx cdk deploy --app cdk.out --context region=${{ matrix.region }} --parameters DeployStage="${{ inputs.stage }}" --parameters HasARM64Support=${{ matrix.has_arm64_support }} 'CanaryV2Stack' --require-approval never --verbose - update_v2_layer_arn_docs: + update_v2_layer_arn_references: needs: deploy-cdk-stack if: ${{ inputs.stage == 'PROD' }} - # lower privilege propagated from parent workflow (publish_v2_layer.yml) + outputs: + temp_branch: ${{ steps.create-pr.outputs.temp_branch }} + runs-on: ubuntu-latest permissions: + # lower privilege propagated from parent workflow (publish_v2_layer.yml) contents: write pull-requests: write + id-token: none pages: none + steps: + - name: Checkout repository # reusable workflows start clean, so we need to checkout again + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + fetch-depth: 0 + - name: Download CDK layer artifact + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: cdk-layer-stack + path: cdk-layer-stack/ + - name: Replace layer versions in documentation + run: | + ls -la cdk-layer-stack/ + ./layer/scripts/update_layer_arn.sh cdk-layer-stack + - name: Create PR + id: create-pr + uses: ./.github/actions/create-pr + with: + files: "docs/index.md examples" + temp_branch_prefix: "ci-layer-docs" + pull_request_title: "chore(ci): layer docs update" + github_token: ${{ secrets.GITHUB_TOKEN }} + + # separate job to keep permissions tight + # so we create a PR with the latest layer changes to a temp branch + # then deploy docs from the temp branch + # otherwise we could have a release for 2.16.0 docs with Layer ARN pointing to 2.15.0 + # because we haven't merged the PR yet + publish_v2_layer_arn_docs: + needs: update_v2_layer_arn_references + permissions: + # lower privilege propagated from parent workflow (reusable_deploy_v2_layer_stack.yml) + contents: write + pages: write + pull-requests: none id-token: none - uses: ./.github/workflows/reusable_update_v2_layer_arn_docs.yml + uses: ./.github/workflows/reusable_publish_docs.yml with: - latest_published_version: ${{ inputs.latest_published_version }} + version: develop + alias: stage + git_ref: ${{ needs.update_v2_layer_arn_references.outputs.temp_branch }} diff --git a/.github/workflows/reusable_publish_docs.yml b/.github/workflows/reusable_publish_docs.yml index 9359229230f..eb0d8f8598a 100644 --- a/.github/workflows/reusable_publish_docs.yml +++ b/.github/workflows/reusable_publish_docs.yml @@ -1,7 +1,6 @@ name: Reusable publish documentation env: - BRANCH: develop ORIGIN: awslabs/aws-lambda-powertools-python on: @@ -20,6 +19,11 @@ on: required: false default: false type: boolean + git_ref: + description: "Branch or commit ID to checkout from" + required: false + type: string + default: develop permissions: contents: write @@ -36,6 +40,7 @@ jobs: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 with: fetch-depth: 0 + ref: ${{ inputs.git_ref }} - name: Install poetry run: pipx install poetry - name: Set up Python @@ -56,6 +61,8 @@ jobs: git config pull.rebase true git config remote.origin.url >&- || git remote add origin https://github.com/"$ORIGIN" git pull origin "$BRANCH" + env: + BRANCH: ${{ inputs.git_ref }} - name: Build docs website and API reference env: VERSION: ${{ inputs.version }} diff --git a/.github/workflows/reusable_update_v2_layer_arn_docs.yml b/.github/workflows/reusable_update_v2_layer_arn_docs.yml deleted file mode 100644 index 68ad6efb9ce..00000000000 --- a/.github/workflows/reusable_update_v2_layer_arn_docs.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Update V2 Layer ARN Docs - -on: - workflow_call: - inputs: - latest_published_version: - description: "Latest PyPi published version to rebuild latest docs for, e.g. v2.0.0" - type: string - required: true - -permissions: - # lower privilege propagated from parent workflow (reusable_deploy_v2_layer_stack.yml) - contents: write - pull-requests: write - pages: none - id-token: none - -env: - BRANCH: develop - -jobs: - publish_v2_layer_arn: - # Force Github action to run only a single job at a time (based on the group name) - # This is to prevent race-condition and inconsistencies with changelog push - concurrency: - group: changelog-build - runs-on: ubuntu-latest - steps: - - name: Checkout repository # reusable workflows start clean, so we need to checkout again - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - with: - fetch-depth: 0 - - name: Git client setup and refresh tip - run: | - git config user.name "Release bot" - git config user.email "aws-devax-open-source@amazon.com" - git config pull.rebase true - git config remote.origin.url >&- || git remote add origin https://github.com/"${origin}" # Git Detached mode (release notes) doesn't have origin - git pull origin "${BRANCH}" - - name: Download CDK layer artifact - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: cdk-layer-stack - path: cdk-layer-stack/ - - name: Replace layer versions in documentation - run: | - ls -la cdk-layer-stack/ - ./layer/scripts/update_layer_arn.sh cdk-layer-stack - - name: Create PR - run: bash .github/scripts/create_pr_for_staged_changes.sh docs/index.md examples - env: - PR_TITLE: "docs(layers): update v2 layer ARN on documentation" - TEMP_BRANCH_PREFIX: "ci-docs" - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cleanup orphaned branch - if: failure() - run: git push origin --delete "${TEMP_BRANCH_PREFIX}-${GITHUB_RUN_ID}" || echo "Must have failed before creating temporary branch; no cleanup needed." - env: - TEMP_BRANCH_PREFIX: "ci-docs" - GITHUB_RUN_ID: ${{ github.run_id }} From d27134fa054ec45579e679b05b363d0ae88e350f Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 10 May 2023 19:26:19 +0200 Subject: [PATCH 9/9] chore: use new custom action to create PR instead --- .github/workflows/release.yml | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a213fbacdc4..c0ccbd01876 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,9 +8,10 @@ name: Release # 2. Bump package version, build release artifact, and generate latest Changelog # 3. Publish package to PyPi prod repository using cached artifact # 4. Kick off Layers pipeline to compile and publish latest version -# 5. Create PR to updates documentation to use the latest Layer ARN for all commercial regions -# 6. Builds a new user guide and API docs with release version; update /latest pointing to newly released version -# 7. Close all issues labeled "pending-release" and notify customers about the release +# 5. Updates latest document with latest Layer ARN for all commercial regions +# 6. Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged +# 7. Builds a new user guide and API docs with release version; update /latest pointing to newly released version +# 8. Close all issues labeled "pending-release" and notify customers about the release # # === Manual activities === # @@ -158,29 +159,17 @@ jobs: RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - - name: Git client setup and refresh tip - run: | - git config user.name "Release bot" - git config user.email "aws-devax-open-source@amazon.com" - git config pull.rebase true - git config remote.origin.url >&- - git pull origin "${BRANCH}" - - name: Bump package version id: versioning run: poetry version "${RELEASE_VERSION}" - name: Create PR - run: bash .github/scripts/create_pr_for_staged_changes.sh pyproject.toml - env: - PR_TITLE: "chore(ci): bump version to ${{ env.RELEASE_VERSION }}" - TEMP_BRANCH_PREFIX: "ci-bump" - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Cleanup orphaned branch - if: failure() - run: git push origin --delete "${TEMP_BRANCH_PREFIX}-${GITHUB_RUN_ID}" || echo "Must have failed before creating temporary branch; no cleanup needed." - env: - TEMP_BRANCH_PREFIX: "ci-bump" - GITHUB_RUN_ID: ${{ github.run_id }} + id: create-pr + uses: ./.github/actions/create-pr + with: + files: "pyproject.toml" + temp_branch_prefix: "ci-bump" + pull_request_title: "chore(ci): bump version to ${{ env.RELEASE_VERSION }}" + github_token: ${{ secrets.GITHUB_TOKEN }} notify_version: needs: [build, release, publish_layer]