diff --git a/.github/actions/download-artifact/action.yml b/.github/actions/download-artifact/action.yml new file mode 100644 index 00000000000..ef938ddb684 --- /dev/null +++ b/.github/actions/download-artifact/action.yml @@ -0,0 +1,58 @@ +name: Download artifact +description: Wrapper around GitHub's official action, with additional extraction before download + +# PROCESS +# +# 1. Downloads artifact using actions/download-artifact action +# 2. Extracts and overwrites tarball previously uploaded +# 3. Remove archive after extraction + +# NOTES +# +# Upload-artifact and download-artifact takes ~2m40s to upload 8MB +# so this is custom action cuts down the entire operation to 1s +# by uploading/extracting a tarball while relying on the official upload-artifact/download-artifact actions +# + +# USAGE +# +# NOTE: Meant to be used with ./.github/actions/upload-artifact +# +# - name: Restore sealed source code +# uses: ./.github/actions/download-artifact +# with: +# name: ${{ needs.seal.outputs.INTEGRITY_HASH }} +# path: . + +# https://github.com/actions/download-artifact/blob/main/action.yml +inputs: + name: + description: Artifact name + required: true + path: + description: Destination path. By default, it will download to the current working directory. + required: false + default: . + +runs: + using: composite + steps: + - name: Download artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: ${{ inputs.name }} + path: ${{ inputs.path }} + + - name: Extract artifacts + run: tar -xvf "${ARCHIVE}" + env: + ARCHIVE: ${{ inputs.name }}.tar + shell: bash + working-directory: ${{ inputs.path }} + + - name: Remove archive + run: rm -f "${ARCHIVE}" + env: + ARCHIVE: ${{ inputs.name }}.tar + shell: bash + working-directory: ${{ inputs.path }} diff --git a/.github/actions/upload-artifact/action.yml b/.github/actions/upload-artifact/action.yml new file mode 100644 index 00000000000..ffa18cc0723 --- /dev/null +++ b/.github/actions/upload-artifact/action.yml @@ -0,0 +1,82 @@ +name: Upload artifact +description: Wrapper around GitHub's official action, with additional archiving before upload + +# PROCESS +# +# 1. Creates tarball excluding .git files +# 2. Uploads tarball using actions/upload-artifact action, fail CI job if no file is found +# 3. Remove archive after uploading it. + +# NOTES +# +# Upload-artifact and download-artifact takes ~2m40s to upload 8MB +# so this is custom action cuts down the entire operation to 1s +# by uploading/extracting a tarball while relying on the official upload-artifact/download-artifact actions +# + +# USAGE +# +# NOTE: Meant to be used with ./.github/actions/download-artifact +# +# - name: Upload sealed source code +# uses: ./.github/actions/upload-artifact +# with: +# name: ${{ steps.integrity.outputs.INTEGRITY_HASH }} +# path: . + +# https://github.com/actions/upload-artifact/blob/main/action.yml +inputs: + name: + description: Artifact name + required: true + path: + description: > + A file, directory or wildcard pattern that describes what to upload. + + You can pass multiple paths separated by space (e.g., dir1 dir2 file.txt). + + Paths and wildcard patterns must be tar command compatible. + required: true + retention-days: + description: > + Artifact retention in days. By default 1 day, max of 90 days, and 0 honours default repo retention. + + You can change max days in the repository settings. + required: false + default: "1" + if-no-files-found: + description: > + Action to perform if no files are found: warn, error, ignore. By default, it fails fast with 'error'. + + Options: + warn: Output a warning but do not fail the action + error: Fail the action with an error message + ignore: Do not output any warnings or errors, the action does not fail + required: false + default: error + +runs: + using: composite + steps: + - name: Archive artifacts + run: | + tar --exclude-vcs \ + -cvf "${ARCHIVE}" "${PATH_TO_ARCHIVE}" + env: + ARCHIVE: ${{ inputs.name }}.tar + PATH_TO_ARCHIVE: ${{ inputs.path }} + shell: bash + + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + if-no-files-found: ${{ inputs.if-no-files-found }} + name: ${{ inputs.name }} + path: ${{ inputs.name }}.tar + retention-days: ${{ inputs.retention-days }} + + - name: Remove archive + run: rm -f "${ARCHIVE}" + env: + ARCHIVE: ${{ inputs.name }}.tar + shell: bash diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 789104dd6db..d5f22affe18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,15 +4,17 @@ name: Release # # === Automated activities === # -# 1. Run tests, linting, security and complexity base line -# 2. Bump package version and build release artifact -# 3. Publish package to PyPi prod repository using cached artifact -# 4. Compile Layer and kick off pipeline for beta, prod, and canary releases -# 5. Update docs with latest Layer ARNs and Changelog -# 6. Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged -# 7. Builds a new user guide and API docs with release version; update /latest pointing to newly released version -# 8. Create PR to update package version on trunk -# 9. Close all issues labeled "pending-release" and notify customers about the release +# 1. [Seal] Bump to release version and export source code with integrity hash +# 2. [Quality check] Restore sealed source code, run tests, linting, security and complexity base line +# 3. [Build] Restore sealed source code, create and export hashed build artifact for PyPi release (wheel, tarball) +# 4. [Release] Restore built artifact, and publish package to PyPi prod repository +# 5. [Create Tag] Restore sealed source code, and create a new git tag using released version +# 6. [PR to bump version] Restore sealed source code, and create a PR to update trunk with latest released project metadata +# 7. [Publish Layer] Compile Layer and kick off pipeline for beta, prod, and canary releases +# 8. [Publish Layer] Update docs with latest Layer ARNs and Changelog +# 9. [Publish Layer] Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged +# 10. [Publish Layer] Builds a new user guide and API docs with release version; update /latest pointing to newly released version +# 11. [Post release] Close all issues labeled "pending-release" and notify customers about the release # # === Manual activities === # @@ -23,8 +25,8 @@ name: Release # See MAINTAINERS.md "Releasing a new version" for release mechanisms env: - BRANCH: develop - ORIGIN: awslabs/aws-lambda-powertools-python + RELEASE_COMMIT: ${{ github.sha }} + RELEASE_TAG_VERSION: ${{ inputs.version_to_publish }} on: workflow_dispatch: @@ -50,72 +52,191 @@ on: required: false jobs: - build: - runs-on: aws-lambda-powertools_ubuntu-latest_4-core + + # This job bumps the package version to the release version + # creates an integrity hash from the source code + # uploads the artifact with the integrity hash as the key name + # so subsequent jobs can restore from a trusted point in time to prevent tampering + seal: + runs-on: ubuntu-latest permissions: contents: read outputs: - RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }} + SOURCE_CODE_HASH: ${{ steps.integrity.outputs.SOURCE_CODE_HASH }} + RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }} + steps: + - name: Export release version + id: release_version + # transform tag format `v` + run: | + RELEASE_VERSION="${RELEASE_TAG_VERSION:1}" + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ env.RELEASE_COMMIT }} + + # We use a pinned version of Poetry to be certain it won't modify source code before we create a hash + - name: Install poetry + run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 + + - name: Bump package version + id: versioning + run: poetry version "${RELEASE_VERSION}" + env: + RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION}} + + - name: Create integrity hash + id: integrity + run: echo "SOURCE_CODE_HASH=${HASH}" >> "$GITHUB_OUTPUT" + env: + # paths to hash and why they're important to protect + # + # aws_lambda_powertools/ - source code + # pyproject.toml - project metadata + # poetry.lock - project dependencies + # layer/ - layer infrastructure and pipeline + # .github/ - github scripts and actions used in the release + # docs/ - user guide documentation + # examples/ - user guide code snippets + HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + + - name: Upload sealed source code + uses: ./.github/actions/upload-artifact + with: + name: source-${{ steps.integrity.outputs.SOURCE_CODE_HASH }} + path: . + + + # This job runs our automated test suite, complexity and security baselines + # it ensures previously merged have been tested as part of the pull request process + # + # NOTE + # + # we don't upload the artifact after testing to prevent any tampering of our source code dependencies + quality_check: + needs: seal + runs-on: ubuntu-latest + permissions: + contents: read env: - RELEASE_TAG_VERSION: ${{ inputs.version_to_publish }} + SOURCE_INTEGRITY_HASH: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} steps: + # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev) - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 with: - fetch-depth: 0 + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: source-${{ env.SOURCE_INTEGRITY_HASH }} + path: . + + - name: Debug cache restore + run: cat pyproject.toml + - name: Install poetry - run: pipx install poetry + run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 - name: Set up Python uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0 with: python-version: "3.10" cache: "poetry" - - name: Set release notes tag - id: release_version - # transform tag format `v> "$GITHUB_ENV" - echo "RELEASE_VERSION=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" - name: Install dependencies run: make dev - name: Run all tests, linting and baselines - if: ${{ !inputs.skip_code_quality }} run: make pr - - name: Bump package version - id: versioning - run: poetry version "${RELEASE_VERSION}" + + # This job creates a release artifact (tar.gz, wheel) + # it checks out code from release commit for custom actions to work + # then restores the sealed source code (overwrites any potential tampering) + # it's done separately from release job to enforce least privilege. + # We export just the final build artifact for release (release-) + build: + runs-on: ubuntu-latest + needs: [quality_check, seal] + permissions: + contents: read + outputs: + BUILD_INTEGRITY_HASH: ${{ steps.integrity.outputs.BUILD_INTEGRITY_HASH }} + env: + SOURCE_INTEGRITY_HASH: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} + steps: + # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev) + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: source-${{ env.SOURCE_INTEGRITY_HASH }} + path: . + + - name: Install poetry + run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 + - name: Set up Python + uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0 + with: + python-version: "3.10" + cache: "poetry" + - name: Build python package and wheel run: poetry build - - name: Cache release artifact - id: cache-release-build - uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + # NOTE: Ran out of time to create a composite action out of this + # because GitHub Action inputs do not support arrays and it became fragile to join multiple strings then split + # keeping these hard coded for now until we have a cleaner way to reuse files/dirs we want to hash + - name: Source code tampering check + run: test "${SOURCE_INTEGRITY_HASH}" = "${CURRENT_HASH}" || exit 1 + env: + CURRENT_HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + + - name: Create integrity hash for build artifact + id: integrity + run: echo "BUILD_INTEGRITY_HASH=${HASH}" >> "$GITHUB_OUTPUT" + env: + # paths to hash and why they're important to protect + # + # dist/ - package distribution build + HASH: ${{ hashFiles('dist/**')}} + + - name: Upload build artifact + uses: ./.github/actions/upload-artifact with: + name: build-${{ steps.integrity.outputs.BUILD_INTEGRITY_HASH}} path: dist/ - # NOTE: cache key uses a hash of (Runner OS + Version to be released + Deps) - # since a new release might not change a dependency but version - # otherwise we might accidentally reuse a previously cached artifact for a newer release. - # The reason we don't add pyproject.toml here is to avoid racing conditions - # where git checkout might happen too fast and doesn't pick up the latest version - # and also future-proof for when we switch to protected branch and update via PR - key: ${{ runner.os }}-${{ env.RELEASE_VERSION }}-${{ hashFiles('**/poetry.lock') }} + # This job uses release artifact to publish to PyPi + # it exchanges JWT tokens with GitHub to obtain PyPi credentials + # since it's already registered as a Trusted Publisher. + # It uses the sealed build artifact (.whl, .tar.gz) to release it release: - needs: build + needs: [build, seal] environment: release - runs-on: aws-lambda-powertools_ubuntu-latest_4-core + runs-on: ubuntu-latest permissions: id-token: write # OIDC for PyPi Trusted Publisher feature env: - RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} + RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} + BUILD_INTEGRITY_HASH: ${{ needs.build.outputs.BUILD_INTEGRITY_HASH }} steps: + # NOTE: we need actions/checkout in order to use our local actions (e.g., ./.github/actions) - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - - name: Restore release artifact from cache - id: restore-release-build - uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 with: - path: dist/ - key: ${{ runner.os }}-${{ env.RELEASE_VERSION }}-${{ hashFiles('**/poetry.lock') }} + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed build + uses: ./.github/actions/download-artifact + with: + name: build-${{ env.BUILD_INTEGRITY_HASH }} + path: . + + - name: Source code tampering check + run: test "${BUILD_INTEGRITY_HASH}" = "${CURRENT_HASH}" || exit 1 + env: + CURRENT_HASH: ${{ hashFiles('dist/**')}} - name: Upload to PyPi prod if: ${{ !inputs.skip_pypi }} @@ -128,65 +249,118 @@ jobs: # with: # repository-url: https://test.pypi.org/legacy/ + # We create a Git Tag using our release version (e.g., v2.16.0) + # using our sealed source code we created earlier. + # Because we bumped version of our project as part of CI + # we need to add this into git before pushing the tag + # otherwise the release commit will be used as the basis for the tag. + # Later, we create a PR to update trunk with our newest release version (e.g., bump_version job) create_tag: - needs: [build, release] + needs: [release, seal] runs-on: ubuntu-latest permissions: contents: write env: - RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} + SOURCE_INTEGRITY_HASH: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} steps: + # NOTE: we need actions/checkout to authenticate and configure git first - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: source-${{ env.SOURCE_INTEGRITY_HASH }} + path: . + + # NOTE: Ran out of time to create a composite action out of this + # because GitHub Action inputs do not support arrays and it became fragile when making it reusable with strings + # keeping these hard coded for now until we have a cleaner way to reuse files/dirs we want to hash + - name: Source code tampering check + run: test "${SOURCE_INTEGRITY_HASH}" = "${CURRENT_HASH}" || exit 1 + env: + CURRENT_HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + - id: setup-git name: Git client setup and refresh tip run: | git config user.name "Powertools bot" git config user.email "aws-lambda-powertools-feedback@amazon.com" git config remote.origin.url >&- + - name: Create Git Tag run: | + git add pyproject.toml + git commit -m "chore: version bump" git tag -a v"${RELEASE_VERSION}" -m "release_version: v${RELEASE_VERSION}" git push origin v"${RELEASE_VERSION}" + env: + RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} - # NOTE: Watch out for the depth limit of 4 nested workflow_calls. - # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack - publish_layer: - needs: [build, release, create_tag] - secrets: inherit - permissions: - id-token: write - contents: write - pages: write - pull-requests: write - uses: ./.github/workflows/publish_v2_layer.yml - with: - latest_published_version: ${{ needs.build.outputs.RELEASE_VERSION }} - pre_release: ${{ inputs.pre_release }} - + # Creates a PR with the latest version we've just released + # since our trunk is protected against any direct pushes from automation bump_version: - needs: [build, release] + needs: [release, seal] permissions: contents: write # create-pr action creates a temporary branch pull-requests: write # create-pr action creates a PR using the temporary branch runs-on: ubuntu-latest env: - RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} + SOURCE_INTEGRITY_HASH: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} steps: + # NOTE: we need actions/checkout to authenticate and configure git first - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - - name: Bump package version - id: versioning - run: poetry version "${RELEASE_VERSION}" + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: source-${{ env.SOURCE_INTEGRITY_HASH }} + path: . + + # NOTE: Ran out of time to create a composite action out of this + # because GitHub Action inputs do not support arrays and it became fragile when making it reusable with strings + # keeping these hard coded for now until we have a cleaner way to reuse files/dirs we want to hash + - name: Source code tampering check + run: test "${SOURCE_INTEGRITY_HASH}" = "${CURRENT_HASH}" || exit 1 + env: + CURRENT_HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + - name: Create PR id: create-pr uses: ./.github/actions/create-pr with: files: "pyproject.toml" temp_branch_prefix: "ci-bump" - pull_request_title: "chore(ci): bump version to ${{ env.RELEASE_VERSION }}" + pull_request_title: "chore(ci): bump version to ${{ needs.seal.outputs.RELEASE_VERSION }}" github_token: ${{ secrets.GITHUB_TOKEN }} + # This job compiles a Lambda Layer optimized for space and speed (e.g., Cython) + # It then deploys to Layer's Beta and Prod account, including SAR Beta and Prod account. + # It uses canaries to attest Layers can be used and imported between stages. + # Lastly, it updates our documentation with the latest Layer ARN for all regions + # + # NOTE + # + # Watch out for the depth limit of 4 nested workflow_calls. + # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack + publish_layer: + needs: [seal, release, create_tag] + secrets: inherit + permissions: + id-token: write + contents: write + pages: write + pull-requests: write + uses: ./.github/workflows/publish_v2_layer.yml + with: + latest_published_version: ${{ needs.seal.outputs.RELEASE_VERSION }} + pre_release: ${{ inputs.pre_release }} + post_release: - needs: [build, release, publish_layer] + needs: [seal, release, publish_layer] permissions: contents: read issues: write @@ -194,9 +368,11 @@ jobs: pull-requests: write runs-on: ubuntu-latest env: - RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} + RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ env.RELEASE_COMMIT }} - name: Close issues related to this release uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1 with: