diff --git a/.github/actions/create-pr/create_pr_for_staged_changes.sh b/.github/actions/create-pr/create_pr_for_staged_changes.sh index 2f32ab24342..99bcd22b97d 100755 --- a/.github/actions/create-pr/create_pr_for_staged_changes.sh +++ b/.github/actions/create-pr/create_pr_for_staged_changes.sh @@ -3,7 +3,7 @@ set -uo pipefail # prevent accessing unset env vars, prevent masking pipeline er #docs #title :create_pr_for_staged_changes.sh -#description :This script will create a PR for staged changes, detect and close duplicate PRs. +#description :This script will create a PR for staged changes, detect and close duplicate PRs. All PRs will be omitted from Release Notes and Changelogs #author :@heitorlessa #date :May 8th 2023 #version :0.1 @@ -61,6 +61,8 @@ function set_environment_variables() { export readonly PR_BODY="This is an automated PR created from the following workflow" export readonly FILENAME=".github/scripts/$(basename "$0")" export readonly NO_DUPLICATES_MESSAGE="No duplicated PRs found" + export readonly SKIP_LABEL="skip-changelog" + end_span } @@ -86,7 +88,8 @@ function create_temporary_branch_with_changes() { function create_pr() { start_span "Creating PR against ${TEMP_BRANCH} branch" - NEW_PR_URL=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}: ${WORKFLOW_URL}" --base "${BASE_BRANCH}" || error "Failed to create PR") # e.g, https://github.com/awslabs/aws-lambda-powertools/pull/13 + # TODO: create label + NEW_PR_URL=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}: ${WORKFLOW_URL}" --base "${BASE_BRANCH}" --label "${SKIP_LABEL}" || error "Failed to create PR") # e.g, https://github.com/awslabs/aws-lambda-powertools/pull/13 # greedy remove any string until the last URL path, including the last '/'. https://opensource.com/article/17/6/bash-parameter-expansion debug "Extracing PR Number from PR URL: "${NEW_PR_URL}"" diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 7820678e813..dcdc43a6e2c 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -1,10 +1,5 @@ name: Deploy v2 layer to all regions -permissions: - id-token: write - contents: write - pages: write - on: workflow_dispatch: inputs: @@ -31,7 +26,11 @@ on: jobs: build-layer: permissions: + # lower privilege propagated from parent workflow (release.yml) contents: read + id-token: none + pages: none + pull-requests: none runs-on: aws-lambda-powertools_ubuntu-latest_8-core defaults: run: @@ -87,6 +86,12 @@ jobs: beta: needs: build-layer + # lower privilege propagated from parent workflow (release.yml) + permissions: + id-token: write + contents: read + pages: write # docs will be updated with latest Layer ARNs + pull-requests: write # creation-action will create a PR with Layer ARN updates uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml secrets: inherit with: @@ -97,6 +102,12 @@ jobs: prod: needs: beta + # lower privilege propagated from parent workflow (release.yml) + permissions: + id-token: write + contents: read + pages: write # docs will be updated with latest Layer ARNs + pull-requests: write # creation-action will create a PR with Layer ARN updates uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml secrets: inherit with: @@ -107,6 +118,12 @@ jobs: sar-beta: needs: build-layer + permissions: + # lower privilege propagated from parent workflow (release.yml) + id-token: write + contents: read + pull-requests: none + pages: none uses: ./.github/workflows/reusable_deploy_v2_sar.yml secrets: inherit with: @@ -117,6 +134,12 @@ jobs: sar-prod: needs: [build-layer, sar-beta] + permissions: + # lower privilege propagated from parent workflow (release.yml) + id-token: write + contents: read + pull-requests: none + pages: none uses: ./.github/workflows/reusable_deploy_v2_sar.yml secrets: inherit with: @@ -125,10 +148,62 @@ jobs: environment: "layer-prod" package-version: ${{ inputs.latest_published_version }} + # Updating the documentation with the latest Layer ARNs is a two-phase process + # + # 1. Update layer ARNs with latest deployed locally and create a PR with these changes + # 2. Pull from temporary branch with these changes and update the docs we're releasing + # + # This keeps our permissions tight and we don't run into a conflict, + # where a new release creates a new doc (2.16.0) while layers are still pointing to 2.15 + # because the PR has to be merged while release process is running + + update_v2_layer_arn_docs: + needs: prod + outputs: + temp_branch: ${{ steps.create-pr.outputs.temp_branch }} + runs-on: ubuntu-latest + permissions: + # lower privilege propagated from parent workflow (release.yml) + contents: write + pull-requests: write + id-token: none + pages: none + steps: + - name: Checkout repository # reusable workflows start clean, so we need to checkout again + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + fetch-depth: 0 + - name: Download CDK layer artifact + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: cdk-layer-stack + path: cdk-layer-stack/ + - name: Replace layer versions in documentation + run: | + ls -la cdk-layer-stack/ + ./layer/scripts/update_layer_arn.sh cdk-layer-stack + # NOTE: It felt unnecessary creating yet another PR to update changelog w/ latest tag + # since this is the only step in the release where we update docs from a temp branch + - name: Update changelog with latest tag + run: make changelog + - name: Create PR + id: create-pr + uses: ./.github/actions/create-pr + with: + files: "docs/index.md examples CHANGELOG.md" + temp_branch_prefix: "ci-layer-docs" + pull_request_title: "chore(ci): layer docs update" + github_token: ${{ secrets.GITHUB_TOKEN }} + + prepare_docs_alias: runs-on: ubuntu-latest permissions: + # lower privilege propagated from parent workflow (release.yml) contents: read + pages: none + id-token: none + pull-requests: none outputs: DOCS_ALIAS: ${{ steps.set-alias.outputs.DOCS_ALIAS }} steps: @@ -141,13 +216,16 @@ jobs: fi echo DOCS_ALIAS="$DOCS_ALIAS" >> "$GITHUB_OUTPUT" - release-docs: - needs: [prod, prepare_docs_alias] + release_docs: + needs: [update_v2_layer_arn_docs, prepare_docs_alias] permissions: + # lower privilege propagated from parent workflow (release.yml) contents: write pages: write + pull-requests: none + id-token: none uses: ./.github/workflows/reusable_publish_docs.yml with: version: ${{ inputs.latest_published_version }} alias: ${{ needs.prepare_docs_alias.outputs.DOCS_ALIAS }} - detached_mode: true + git_ref: ${{ needs.update_v2_layer_arn_docs.outputs.temp_branch }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f062baa01b2..789104dd6db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,12 +5,14 @@ name: Release # === Automated activities === # # 1. Run tests, linting, security and complexity base line -# 2. Bump package version, build release artifact, and generate latest Changelog +# 2. Bump package version and build release artifact # 3. Publish package to PyPi prod repository using cached artifact -# 4. Kick off Layers pipeline to compile and publish latest version -# 5. Updates documentation to use the latest Layer ARN for all commercial regions -# 6. Builds a new user guide and API docs with release version; update /latest pointing to newly released version -# 7. Close all issues labeled "pending-release" and notify customers about the release +# 4. Compile Layer and kick off pipeline for beta, prod, and canary releases +# 5. Update docs with latest Layer ARNs and Changelog +# 6. Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged +# 7. Builds a new user guide and API docs with release version; update /latest pointing to newly released version +# 8. Create PR to update package version on trunk +# 9. Close all issues labeled "pending-release" and notify customers about the release # # === Manual activities === # @@ -126,15 +128,36 @@ jobs: # with: # repository-url: https://test.pypi.org/legacy/ + create_tag: + needs: [build, release] + runs-on: ubuntu-latest + permissions: + contents: write + env: + RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + - id: setup-git + name: Git client setup and refresh tip + run: | + git config user.name "Powertools bot" + git config user.email "aws-lambda-powertools-feedback@amazon.com" + git config remote.origin.url >&- + - name: Create Git Tag + run: | + git tag -a v"${RELEASE_VERSION}" -m "release_version: v${RELEASE_VERSION}" + git push origin v"${RELEASE_VERSION}" + # NOTE: Watch out for the depth limit of 4 nested workflow_calls. - # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack -> reusable_update_v2_layer_arn_docs + # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack publish_layer: - needs: [build, release] + needs: [build, release, create_tag] secrets: inherit permissions: id-token: write contents: write pages: write + pull-requests: write uses: ./.github/workflows/publish_v2_layer.yml with: latest_published_version: ${{ needs.build.outputs.RELEASE_VERSION }} diff --git a/.github/workflows/reusable_deploy_v2_layer_stack.yml b/.github/workflows/reusable_deploy_v2_layer_stack.yml index 912849c2e2c..b1113ff3e40 100644 --- a/.github/workflows/reusable_deploy_v2_layer_stack.yml +++ b/.github/workflows/reusable_deploy_v2_layer_stack.yml @@ -1,9 +1,5 @@ name: Deploy CDK Layer v2 stack -permissions: - id-token: write - contents: write - on: workflow_call: inputs: @@ -28,6 +24,12 @@ jobs: deploy-cdk-stack: runs-on: ubuntu-latest environment: ${{ inputs.environment }} + # lower privilege propagated from parent workflow (publish_v2_layer.yml) + permissions: + id-token: write + pull-requests: none + contents: read + pages: none defaults: run: working-directory: ./layer @@ -149,10 +151,3 @@ jobs: retention-days: 1 - name: CDK Deploy Canary run: npx cdk deploy --app cdk.out --context region=${{ matrix.region }} --parameters DeployStage="${{ inputs.stage }}" --parameters HasARM64Support=${{ matrix.has_arm64_support }} 'CanaryV2Stack' --require-approval never --verbose - - update_v2_layer_arn_docs: - needs: deploy-cdk-stack - if: ${{ inputs.stage == 'PROD' }} - uses: ./.github/workflows/reusable_update_v2_layer_arn_docs.yml - with: - latest_published_version: ${{ inputs.latest_published_version }} diff --git a/.github/workflows/reusable_publish_docs.yml b/.github/workflows/reusable_publish_docs.yml index 9359229230f..eb0d8f8598a 100644 --- a/.github/workflows/reusable_publish_docs.yml +++ b/.github/workflows/reusable_publish_docs.yml @@ -1,7 +1,6 @@ name: Reusable publish documentation env: - BRANCH: develop ORIGIN: awslabs/aws-lambda-powertools-python on: @@ -20,6 +19,11 @@ on: required: false default: false type: boolean + git_ref: + description: "Branch or commit ID to checkout from" + required: false + type: string + default: develop permissions: contents: write @@ -36,6 +40,7 @@ jobs: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 with: fetch-depth: 0 + ref: ${{ inputs.git_ref }} - name: Install poetry run: pipx install poetry - name: Set up Python @@ -56,6 +61,8 @@ jobs: git config pull.rebase true git config remote.origin.url >&- || git remote add origin https://github.com/"$ORIGIN" git pull origin "$BRANCH" + env: + BRANCH: ${{ inputs.git_ref }} - name: Build docs website and API reference env: VERSION: ${{ inputs.version }} diff --git a/.github/workflows/reusable_update_v2_layer_arn_docs.yml b/.github/workflows/reusable_update_v2_layer_arn_docs.yml deleted file mode 100644 index 5fbf6814dcd..00000000000 --- a/.github/workflows/reusable_update_v2_layer_arn_docs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Update V2 Layer ARN Docs - -on: - workflow_call: - inputs: - latest_published_version: - description: "Latest PyPi published version to rebuild latest docs for, e.g. v2.0.0" - type: string - required: true - -permissions: - contents: write - -env: - BRANCH: develop - -jobs: - publish_v2_layer_arn: - # Force Github action to run only a single job at a time (based on the group name) - # This is to prevent race-condition and inconsistencies with changelog push - concurrency: - group: changelog-build - runs-on: ubuntu-latest - steps: - - name: Checkout repository # reusable workflows start clean, so we need to checkout again - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - with: - fetch-depth: 0 - - name: Git client setup and refresh tip - run: | - git config user.name "Release bot" - git config user.email "aws-devax-open-source@amazon.com" - git config pull.rebase true - git config remote.origin.url >&- || git remote add origin https://github.com/"${origin}" # Git Detached mode (release notes) doesn't have origin - git pull origin "${BRANCH}" - - name: Download CDK layer artifact - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: cdk-layer-stack - path: cdk-layer-stack/ - - name: Replace layer versions in documentation - run: | - ls -la cdk-layer-stack/ - ./layer/scripts/update_layer_arn.sh cdk-layer-stack - - name: Update documentation in trunk - run: | - HAS_CHANGE=$(git status --porcelain) - test -z "${HAS_CHANGE}" && echo "Nothing to update" && exit 0 - git add docs/index.md examples - git commit -m "chore: update v2 layer ARN on documentation" - git pull origin "${BRANCH}" # prevents concurrent branch update failing push - git push origin HEAD:refs/heads/"${BRANCH}" diff --git a/layer/scripts/update_layer_arn.sh b/layer/scripts/update_layer_arn.sh index 0ad3e1617fe..1bbf63c2b88 100755 --- a/layer/scripts/update_layer_arn.sh +++ b/layer/scripts/update_layer_arn.sh @@ -1,6 +1,6 @@ #!/bin/bash -# This script is run during the reusable_update_v2_layer_arn_docs CI job, +# This script is run during the publish_v2_layer.yml CI job, # and it is responsible for replacing the layer ARN in our documentation, # based on the output files generated by CDK when deploying to each pseudo_region. #