chore(ci): increase release automation and limit to one manual step (aws-powertools#1297)

Author: heitorlessa

.chglog/CHANGELOG.tpl.md

@@ -1,10 +1,13 @@
+<!-- changelog is partially generated, so it doesn't follow headings and required structure, so we disable it. -->
+<!-- markdownlint-disable -->
+
 {{ if .Versions -}}
 <a name="unreleased"></a>
-## Unreleased
+# Unreleased
 
 {{ if .Unreleased.CommitGroups -}}
 {{ range .Unreleased.CommitGroups -}}
-### {{ .Title }}
+## {{ .Title }}
 
 {{ range .Commits -}}
 * {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
@@ -17,22 +20,23 @@
 <a name="{{ .Tag.Name }}"></a>
 ## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }}
 {{ range .CommitGroups -}}
-### {{ .Title }}
+
+## {{ .Title }}
 
 {{ range .Commits -}}
 * {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
 {{ end }}
 {{ end -}}
 
 {{- if .RevertCommits -}}
-### Reverts
+## Reverts
 {{ range .RevertCommits -}}
 * {{ .Revert.Header }}
 {{ end }}
 {{ end -}}
 
 {{- if .MergeCommits -}}
-### Pull Requests
+## Pull Requests
 
 {{ range .MergeCommits -}}
 * {{ .Header }}
@@ -41,7 +45,7 @@
 
 {{- if .NoteGroups -}}
 {{ range .NoteGroups -}}
-### {{ .Title }}
+## {{ .Title }}
 {{ range .Notes }}
 {{ .Body }}
 {{ end }}

.chglog/config.yml

@@ -32,3 +32,6 @@ options:
   notes:
     keywords:
       - BREAKING CHANGE
+  # issues:
+  #   prefix:
+  #     - #

.github/scripts/post_release.js

@@ -100,13 +100,13 @@ const notifyRelease = async ({
 
 // context: https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts
 module.exports = async ({ github, context }) => {
-	const { RELEASE_TAG_VERSION } = process.env;
-	console.log(`Running post-release script for ${RELEASE_TAG_VERSION} version`);
+	const { RELEASE_VERSION } = process.env;
+	console.log(`Running post-release script for ${RELEASE_VERSION} version`);
 
 	await notifyRelease({
 		gh_client: github,
 		owner: context.repo.owner,
 		repository: context.repo.repo,
-		release_version: RELEASE_TAG_VERSION,
+		release_version: RELEASE_VERSION,
 	});
 };

.github/workflows/publish.yml

@@ -4,44 +4,38 @@ name: Publish to PyPi
 #
 # === Manual activities ===
 #
-# 1. Document human readable changes in CHANGELOG (pre-generate unreleased changes with `make changelog`)
-# 2. Bump package version using poetry version <major|minor|patch|specific version>
-# 3. Merge version changes to develop branch
-# 4. Edit the current draft release notes
-# 5. If not already set, use `v<new version>` as a tag, and select develop as target branch
+# 1. Edit the current draft release notes
+# 2. If not already set, use `v<new version>` as a tag, e.g., v1.26.4, and select develop as target branch
 #
 # === Automated activities ===
 #
 # 1. Extract release notes tag that was published
-# 2. Ensure release notes tag match what's in CHANGELOG and pyproject
-# 3. Run tests, linting, security and complexity base line
-# 4. Publish package to PyPi test repository
-# 5. Publish package to PyPi prod repository
-# 6. Kick off Lambda Layer pipeline to publish latest version with minimal dependencies as a SAR App
-# 7. Kick off Lambda Layer pipeline to publish latest version with extra dependencies as a SAR App
-# 8. Builds a fresh version of docs including Changelog updates
-# 9. Builds latest documentation for new release, and update latest alias pointing to the new release tag
-# 10. Close and notify all issues labeled "pending-release" about the release details
+# 2. Run tests, linting, security and complexity base line
+# 3. Bump package version and generate latest Changelog
+# 4. Publish package to PyPi test and prod repository
+# 5. Kick off SAR App pipeline to publish latest version with minimal and extra dependencies
+# 6. Builds a new user guide and API docs with release version; update /latest pointing to newly released version
+# 7. Close all issues labeled "pending-release" and notify customers about the release
 
-#
-# === Fallback mechanism due to external failures ===
-#
-# 1. Trigger "Publish to PyPi" workflow manually: https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
-# 2. Use the version released under Releases e.g. v1.13.0
-#
-
-#
-# === Documentation hotfix ===
-#
-# Look for rebuild latest docs workflow
+# See MAINTAINERS.md "Releasing a new version" for release mechanisms
 
 on:
   release:
     types: [published]
+  workflow_dispatch:
+    inputs:
+      version_to_publish:
+        description: "Version to be released in PyPi, Docs, and Lambda Layer, e.g. v1.26.4"
+        default: v1.26.4
+        required: true
 
 jobs:
   release:
     runs-on: ubuntu-latest
+    outputs:
+      RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }}
+    env:
+      RELEASE_TAG_VERSION: ${{ github.event.release.tag_name || inputs.version_to_publish }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -51,17 +45,30 @@ jobs:
         with:
           python-version: "3.8"
       - name: Set release notes tag
+        id: release_version
+        # transform tag format `v<version` to `<version`
         run: |
-          RELEASE_TAG_VERSION=${{ github.event.release.tag_name }}
-          echo "RELEASE_TAG_VERSION=${RELEASE_TAG_VERSION:1}" >> $GITHUB_ENV
-      - name: Ensure new version is also set in pyproject and CHANGELOG
-        run: |
-          grep --regexp "${RELEASE_TAG_VERSION}" CHANGELOG.md
-          grep --regexp "version \= \"${RELEASE_TAG_VERSION}\"" pyproject.toml
+          RELEASE_VERSION=${RELEASE_TAG_VERSION:1}
+          echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV
+          echo "::set-output name=RELEASE_VERSION::${RELEASE_VERSION}"
       - name: Install dependencies
         run: make dev
       - name: Run all tests, linting and baselines
         run: make pr
+      - name: Bump package version
+        run: poetry version ${RELEASE_VERSION}
+      - name: Generate latest CHANGELOG
+        run: make changelog
+      - name: Setup git client
+        run: |
+          git config user.name "Release bot"
+          git config user.email [email protected]
+      - name: Push project metadata and changelog to trunk
+        run: |
+          git add CHANGELOG.md
+          git add pyproject.toml
+          git commit -m "chore(ci): update project with version ${RELEASE_VERSION}"
+          git push origin HEAD:refs/heads/develop
       - name: Build python package and wheel
         run: poetry build
       - name: Upload to PyPi test
@@ -76,39 +83,58 @@ jobs:
           PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
       - name: publish lambda layer in SAR by triggering the internal codepipeline
         run: |
-          aws ssm put-parameter --name "powertools-python-release-version" --value $RELEASE_TAG_VERSION --overwrite
+          aws ssm put-parameter --name "powertools-python-release-version" --value $RELEASE_VERSION --overwrite
           aws codepipeline start-pipeline-execution --name ${{ secrets.CODEPIPELINE_NAME }}
         env:
           # Maintenance: Migrate to new OAuth mechanism
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: eu-west-1
           AWS_DEFAULT_OUTPUT: json
-      - name: Setup doc deploy
+
+  docs:
+    needs: release
+    runs-on: ubuntu-latest
+    env:
+      RELEASE_VERSION: ${{ needs.release.outputs.RELEASE_VERSION }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Setup git client
         run: |
-          git config --global user.name Docs deploy
-          git config --global user.email [email protected]
+          git config user.name "Release bot"
+          git config user.email [email protected]
       - name: Build docs website and API reference
         run: |
-          make release-docs VERSION=${RELEASE_TAG_VERSION} ALIAS="latest"
+          make release-docs VERSION=${RELEASE_VERSION} ALIAS="latest"
           poetry run mike set-default --push latest
       - name: Release API docs to release version
         uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./api
           keep_files: true
-          destination_dir: ${{ env.RELEASE_TAG_VERSION }}/api
+          destination_dir: ${{ env.RELEASE_VERSION }}/api
       - name: Release API docs to latest
         uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./api
           keep_files: true
           destination_dir: latest/api
+
+  post_release:
+    needs: release
+    runs-on: ubuntu-latest
+    env:
+      RELEASE_VERSION: ${{ needs.release.outputs.RELEASE_VERSION }}
+    steps:
+      - uses: actions/checkout@v3
       - name: Close issues related to this release
         uses: actions/github-script@v6
         with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const post_release = require('.github/scripts/post_release.js')
             await post_release({github, context, core})

.github/workflows/python_docs.yml

@@ -1,12 +1,14 @@
 name: Docs
 
+# Maintenance: Create a reusable workflow to be more easily reused across release, push, and doc hot fixes
+# this should include inputs on whether to release API docs, what version to release, and whether to rebuild /latest
+
 on:
   push:
     branches:
       - develop
     paths:
       - "docs/**"
-      - "CHANGELOG.md"
       - "mkdocs.yml"
       - "examples/**"
 

MAINTAINERS.md

@@ -13,8 +13,6 @@
     - [Triage Bug Reports](#triage-bug-reports)
     - [Triage RFCs](#triage-rfcs)
     - [Releasing a new version](#releasing-a-new-version)
-        - [Changelog generation](#changelog-generation)
-        - [Bumping the version](#bumping-the-version)
         - [Drafting release notes](#drafting-release-notes)
     - [Run end to end tests](#run-end-to-end-tests)
     - [Releasing a documentation hotfix](#releasing-a-documentation-hotfix)
@@ -170,23 +168,15 @@ Some examples using our initial and new RFC templates: #92, #94, #95, #991, #122
 
 ### Releasing a new version
 
-> TODO: This is an area we want to increase automation while keeping communication at human level.
+Firstly, make sure the commit history in the `develop` branch **(1)** it's up to date, **(2)** commit messages are semantic, and **(3)** commit messages have their respective area, for example `feat(logger): <change>`, `chore(ci): ...`).
 
-Firstly, make sure you are using the `develop` branch and it is up to date with the origin.
+**Found typos or unclear commit messages?**
 
-There are three main steps to release a new version: Changelog generation, version bumping, and drafting release notes.
+Reword through rebase and push with `--force-with-lease` once you're confident. This will ensure [CHANGELOG](./CHANGELOG.md) is always clear for customers looking to understand what changed in between releases - was that a bug? what new features and for which utility?
 
-#### Changelog generation
+**Looks good, what's next?**
 
-You can pre-generate a temporary CHANGELOG using `make changelog`. This will generate a `TMP_CHANGELOG.md` with all staged changes under the `unreleased` section.
-
-Each unreleased line item is a commit. You can adjust them if you find the commit titles are insufficient to describe their intent. Once you're comfortable, bring these changes to the `CHANGELOG.md` with a new version heading like in previous versions.
-
-#### Bumping the version
-
-Use `poetry version <major|minor|patch|specific version>` to bump the version. For example, you can use `poetry version minor` when releasing a minor version.
-
-NOTE. Make sure both `CHANGELOG` and `pyproject.toml` are committed and pushed to the remote `develop` branch before proceeding.
+The only step is to draft and publish a good release notes, everything else is automated.
 
 #### Drafting release notes
 
@@ -196,28 +186,36 @@ Make sure the `tag` field reflects the new version you're releasing, the target
 
 You'll notice we group all changes based on their [labels](#labels) like `feature`, `bug`, `documentation`, etc.
 
-> **Q: What if there's an incorrect title or grouping?**
+**I spotted a typo or incorrect grouping - how do I fix it?**
 
 Edit the respective PR title and update their [labels](#labels). Then run the [Release Drafter workflow](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/release-drafter.yml) to update the Draft release.
 
-The best part comes now. Replace the placeholder `[Human readable summary of changes]` with what you'd like to communicate to customers what this release is all about. Always put yourself in the customers shoes. For that, these are some questions to keep in mind when drafting your first or future release notes:
+**All looking good, what's next?**
+
+The best part comes now. Replace the placeholder `[Human readable summary of changes]` with what you'd like to communicate to customers what this release is all about. Rule of thumb: always put yourself in the customers shoes.
+
+These are some questions to keep in mind when drafting your first or future release notes:
 
 - Can customers understand at a high level what changed in this release?
 - Is there a link to the documentation where they can read more about each main change?
-- Are there any graphics or code snippets that can enhance readability?
+- Are there any graphics or [code snippets](carbon.now.sh/) that can enhance readability?
 - Are we calling out any key contributor(s) to this release?
     - All contributors are automatically credited, use this as an exceptional case to feature them
 
-Once you're happy, hit `Publish release`. This will kick off the [Publishing workflow](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/publish.yml) and within a few minutes you should see the latest version in PyPi, and all issues labeled as `pending-release` will be notified.
+Once you're happy, hit `Publish release` 🎉🎉🎉.
+
+This will kick off the [Publishing workflow](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/publish.yml) and within a few minutes you should see the latest version in PyPi, and all issues labeled as `pending-release` will be closed and notified.
+
+> TODO: Include information to verify SAR and Lambda Layers deployment; we're still finalizing Lambda Layer automated deployment in GitHub Actions - ping @am29d when in doubt.
 
-> TODO: Wait for @am29d new Lambda Layers pipeline work to complete, then add how Lambda Layers are published
 ### Run end to end tests
 
 In order to run end to end tests you need to install CDK CLI first and bootstrap your account with `cdk bootstrap` command. For additional details follow [documentation](https://docs.aws.amazon.com/cdk/v2/guide/bootstrapping.html).
 
 To run locally, export `AWS_PROFILE` environment variable and run `make e2e tests`. To run from GitHub Actions, use [run-e2e-tests workflow](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/run-e2e-tests.yml) and pick the branch you want to run tests against.
 
 **NOTE**: E2E tests are run as part of each merge to `develop` branch.
+
 ### Releasing a documentation hotfix
 
 You can rebuild the latest documentation without a full release via this [GitHub Actions Workflow](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/rebuild_latest_docs.yml). Choose `Run workflow`, keep `develop` as the branch, and input the latest Powertools version available.

Makefile

@@ -94,8 +94,9 @@ release: pr
 
 changelog:
 	git fetch --tags origin
-	@echo "[+] Pre-generating CHANGELOG for tag: $$(git describe --abbrev=0 --tag)"
-	docker run -v "${PWD}":/workdir quay.io/git-chglog/git-chglog $$(git describe --abbrev=0 --tag).. > TMP_CHANGELOG.md
+	CURRENT_VERSION=$(shell git describe --abbrev=0 --tag) ;\
+	echo "[+] Pre-generating CHANGELOG for tag: $$CURRENT_VERSION" ;\
+	docker run -v "${PWD}":/workdir quay.io/git-chglog/git-chglog > CHANGELOG.md
 
 mypy:
 	poetry run mypy --pretty aws_lambda_powertools