From 7c25cf1faf63c7b3e6cfe5c0eed34d50730068d7 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 22 May 2023 09:42:54 +0200 Subject: [PATCH 1/9] chore: create download-artifact, upload-artifact --- .github/actions/download-artifact/action.yml | 58 ++++++++++++++ .github/actions/upload-artifact/action.yml | 83 ++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 .github/actions/download-artifact/action.yml create mode 100644 .github/actions/upload-artifact/action.yml diff --git a/.github/actions/download-artifact/action.yml b/.github/actions/download-artifact/action.yml new file mode 100644 index 00000000000..85ae351c0e4 --- /dev/null +++ b/.github/actions/download-artifact/action.yml @@ -0,0 +1,58 @@ +name: Download artifact +description: Wrapper around GitHub's official action, with additional extraction before download + +# PROCESS +# +# 1. Downloads artifact using actions/download-artifact action +# 2. Extracts and overwrites tarball previously uploaded +# 3. Remove archive after extraction + +# NOTES +# +# Upload-artifact and download-artifact takes ~2m40s to upload 8MB +# so this is custom action cuts down the entire operation to 1s +# by uploading/extracting a tarball while relying on the official upload-artifact/download-artifact actions +# + +# USAGE +# +# NOTE: Meant to be used with ./.github/actions/upload-artifact +# +# - name: Restore sealed source code +# uses: ./.github/actions/download-artifact +# with: +# name: ${{ needs.seal.outputs.INTEGRITY_HASH }} +# path: . + +# https://github.com/actions/download-artifact/blob/main/action.yml +inputs: + name: + description: Artifact name + required: true + path: + description: Destination path. By default, it will download to the current working directory. + required: false + default: . + +runs: + using: composite + steps: + - name: Download artifacts + uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # v2.1.1 + with: + name: ${{ inputs.name }} + path: ${{ inputs.path }} + + - name: Extract artifacts + run: tar -xvf "${ARCHIVE}" + env: + ARCHIVE: ${{ inputs.name }}.tar + shell: bash + working-directory: ${{ inputs.path }} + + - name: Remove archive + run: rm -f "${ARCHIVE}" + env: + ARCHIVE: ${{ inputs.name }}.tar + shell: bash + working-directory: ${{ inputs.path }} diff --git a/.github/actions/upload-artifact/action.yml b/.github/actions/upload-artifact/action.yml new file mode 100644 index 00000000000..ea6b4bc475a --- /dev/null +++ b/.github/actions/upload-artifact/action.yml @@ -0,0 +1,83 @@ +name: Upload artifact +description: Wrapper around GitHub's official action, with additional archiving before upload + +# PROCESS +# +# 1. Creates tarball excluding .git and *.pyc files +# 2. Uploads tarball using actions/upload-artifact action, fail CI job if no file is found +# 3. Remove archive after uploading it. + +# NOTES +# +# Upload-artifact and download-artifact takes ~2m40s to upload 8MB +# so this is custom action cuts down the entire operation to 1s +# by uploading/extracting a tarball while relying on the official upload-artifact/download-artifact actions +# + +# USAGE +# +# NOTE: Meant to be used with ./.github/actions/download-artifact +# +# - name: Upload sealed source code +# uses: ./.github/actions/upload-artifact +# with: +# name: ${{ steps.integrity.outputs.INTEGRITY_HASH }} +# path: . + +# https://github.com/actions/upload-artifact/blob/main/action.yml +inputs: + name: + description: Artifact name + required: true + path: + description: > + A file, directory or wildcard pattern that describes what to upload. + + You can pass multiple paths separated by space (e.g., dir1 dir2 file.txt). + + Paths and wildcard patterns must be tar command compatible. + required: true + retention-days: + description: > + Artifact retention in days. By default 1 day, max of 90 days, and 0 honours default repo retention. + + You can change max days in the repository settings. + required: false + default: "1" + if-no-files-found: + description: > + Action to perform if no files are found: warn, error, ignore. By default, it fails fast with 'error'. + + Options: + warn: Output a warning but do not fail the action + error: Fail the action with an error message + ignore: Do not output any warnings or errors, the action does not fail + required: false + default: error + +runs: + using: composite + steps: + - name: Archive artifacts + run: | + tar --exclude-vcs \ + --exclude "*.pyc" \ + -cvf "${ARCHIVE}" "${PATH_TO_ARCHIVE}" + env: + ARCHIVE: ${{ inputs.name }}.tar + PATH_TO_ARCHIVE: ${{ inputs.path }} + shell: bash + + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + if-no-files-found: ${{ inputs.if-no-files-found }} + name: ${{ inputs.name }} + path: ${{ inputs.name }}.tar + retention-days: ${{ inputs.retention-days }} + + - name: Remove archive + run: rm -f "${ARCHIVE}" + env: + ARCHIVE: ${{ inputs.name }}.tar + shell: bash From 8663cee17e14a6de60aacac168620d1453f24526 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 22 May 2023 12:00:30 +0200 Subject: [PATCH 2/9] chore: seals source code, separate quality check & build Uses new mechanism to seal source code, run quality checks separately, and always reuses sealed source code in every step to prevent source code tampering. It documents every job purpose, and creates a new faster action to upload and download artifacts. --- .github/workflows/release.yml | 252 +++++++++++++++++++++++++--------- 1 file changed, 187 insertions(+), 65 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 789104dd6db..4be9c290b12 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,8 +23,8 @@ name: Release # See MAINTAINERS.md "Releasing a new version" for release mechanisms env: - BRANCH: develop - ORIGIN: awslabs/aws-lambda-powertools-python + RELEASE_COMMIT: ${{ github.sha }} + RELEASE_TAG_VERSION: ${{ inputs.version_to_publish }} on: workflow_dispatch: @@ -50,72 +50,161 @@ on: required: false jobs: - build: - runs-on: aws-lambda-powertools_ubuntu-latest_4-core + + # This job bumps the package version to the release version + # creates an integrity hash from the source code + # uploads the artifact with the integrity hash as the key name + # so subsequent jobs can restore from a trusted point in time to prevent tampering + seal: + runs-on: ubuntu-latest permissions: contents: read outputs: - RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }} - env: - RELEASE_TAG_VERSION: ${{ inputs.version_to_publish }} + INTEGRITY_HASH: ${{ steps.integrity.outputs.INTEGRITY_HASH }} + RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }} steps: + - name: Export release version + id: release_version + # transform tag format `v` + run: | + RELEASE_VERSION="${RELEASE_TAG_VERSION:1}" + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 with: - fetch-depth: 0 + ref: ${{ env.RELEASE_COMMIT }} + + # We use a pinned version of Poetry to be certain it won't modify source code before we create a hash - name: Install poetry - run: pipx install poetry + run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 + + - name: Bump package version + id: versioning + run: poetry version "${RELEASE_VERSION}" + env: + RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION}} + + - name: Create integrity hash + id: integrity + run: echo "INTEGRITY_HASH=${SOURCE_CODE_HASH}" >> "$GITHUB_OUTPUT" + env: + # paths to hash and why they're important to protect + # + # aws_lambda_powertools/ - source code + # pyproject.toml - project metadata + # poetry.lock - project dependencies + # layer/ - layer infrastructure and pipeline + # .github/ - github scripts and actions used in the release + # docs/ - user guide documentation + # examples/ - user guide code snippets + SOURCE_CODE_HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + + - name: Upload sealed source code + uses: ./.github/actions/upload-artifact + with: + name: ${{ steps.integrity.outputs.INTEGRITY_HASH }} + path: . + + + # This job runs our automated test suite, complexity and security baselines + # it ensures previously merged have been tested as part of the pull request process + # + # NOTE + # + # we don't upload the artifact after tested to prevent no dependencies tamper our source code + quality_check: + needs: seal + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev) + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: ${{ needs.seal.outputs.INTEGRITY_HASH }} + path: . + + - name: Debug cache restore + run: cat pyproject.toml + + - name: Install poetry + run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 - name: Set up Python uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0 with: python-version: "3.10" cache: "poetry" - - name: Set release notes tag - id: release_version - # transform tag format `v> "$GITHUB_ENV" - echo "RELEASE_VERSION=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" - name: Install dependencies run: make dev - name: Run all tests, linting and baselines - if: ${{ !inputs.skip_code_quality }} run: make pr - - name: Bump package version - id: versioning - run: poetry version "${RELEASE_VERSION}" + + # This job creates a release artifact (tar.gz, wheel) + # it checks out code from release commit for custom actions to work + # then restores the sealed source code (overwrites any potential tampering) + # it's done separately from release job to enforce least privilege. + # We export just the final build artifact for release (release-) + build: + runs-on: ubuntu-latest + needs: [quality_check, seal] + permissions: + contents: read + steps: + # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev) + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: ${{ needs.seal.outputs.INTEGRITY_HASH }} + path: . + + - name: Install poetry + run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 + - name: Set up Python + uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0 + with: + python-version: "3.10" + cache: "poetry" + - name: Build python package and wheel run: poetry build - - name: Cache release artifact - id: cache-release-build - uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + - name: Upload build artifact + uses: ./.github/actions/upload-artifact with: + name: release-${{ needs.seal.outputs.INTEGRITY_HASH }} path: dist/ - # NOTE: cache key uses a hash of (Runner OS + Version to be released + Deps) - # since a new release might not change a dependency but version - # otherwise we might accidentally reuse a previously cached artifact for a newer release. - # The reason we don't add pyproject.toml here is to avoid racing conditions - # where git checkout might happen too fast and doesn't pick up the latest version - # and also future-proof for when we switch to protected branch and update via PR - key: ${{ runner.os }}-${{ env.RELEASE_VERSION }}-${{ hashFiles('**/poetry.lock') }} + # This job uses release artifact to publish to PyPi + # it exchanges JWT tokens with GitHub to obtain PyPi credentials + # since it's already registered as a Trusted Publisher. + # It uses the sealed build artifact (.whl, .tar.gz) to release it release: - needs: build + needs: [build, seal] environment: release - runs-on: aws-lambda-powertools_ubuntu-latest_4-core + runs-on: ubuntu-latest permissions: id-token: write # OIDC for PyPi Trusted Publisher feature env: - RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} + RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - - name: Restore release artifact from cache - id: restore-release-build - uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 with: - path: dist/ - key: ${{ runner.os }}-${{ env.RELEASE_VERSION }}-${{ hashFiles('**/poetry.lock') }} + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: release-${{ needs.seal.outputs.INTEGRITY_HASH }} + path: . - name: Upload to PyPi prod if: ${{ !inputs.skip_pypi }} @@ -128,65 +217,98 @@ jobs: # with: # repository-url: https://test.pypi.org/legacy/ + # We create a Git Tag using our release version (e.g., v2.16.0) + # using our sealed source code we created earlier. + # Because we bumped version of our project as part of CI + # we need to add this into git before pushing the tag + # otherwise the release commit will be used as the basis for the tag. + # Later, we create a PR to update trunk with our newest release version (e.g., bump_version job) create_tag: - needs: [build, release] + needs: [release, seal] runs-on: ubuntu-latest permissions: contents: write - env: - RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} steps: + # NOTE: we need actions/checkout to authenticate and configure git first - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: ${{ needs.seal.outputs.INTEGRITY_HASH }} + path: . + - id: setup-git name: Git client setup and refresh tip run: | git config user.name "Powertools bot" git config user.email "aws-lambda-powertools-feedback@amazon.com" git config remote.origin.url >&- + - name: Create Git Tag run: | + git add pyproject.toml + git commit -m "chore: version bump" git tag -a v"${RELEASE_VERSION}" -m "release_version: v${RELEASE_VERSION}" git push origin v"${RELEASE_VERSION}" + env: + RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} - # NOTE: Watch out for the depth limit of 4 nested workflow_calls. - # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack - publish_layer: - needs: [build, release, create_tag] - secrets: inherit - permissions: - id-token: write - contents: write - pages: write - pull-requests: write - uses: ./.github/workflows/publish_v2_layer.yml - with: - latest_published_version: ${{ needs.build.outputs.RELEASE_VERSION }} - pre_release: ${{ inputs.pre_release }} - + # Creates a PR with the latest version we've just released + # since our trunk is protected against any direct pushes from automation bump_version: - needs: [build, release] + needs: [release, seal] permissions: contents: write # create-pr action creates a temporary branch pull-requests: write # create-pr action creates a PR using the temporary branch runs-on: ubuntu-latest - env: - RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} steps: + # NOTE: we need actions/checkout to authenticate and configure git first - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - - name: Bump package version - id: versioning - run: poetry version "${RELEASE_VERSION}" + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/download-artifact + with: + name: ${{ needs.seal.outputs.INTEGRITY_HASH }} + path: . + - name: Create PR id: create-pr uses: ./.github/actions/create-pr with: files: "pyproject.toml" temp_branch_prefix: "ci-bump" - pull_request_title: "chore(ci): bump version to ${{ env.RELEASE_VERSION }}" + pull_request_title: "chore(ci): bump version to ${{ needs.seal.outputs.RELEASE_VERSION }}" github_token: ${{ secrets.GITHUB_TOKEN }} + # This job compiles a Lambda Layer optimized for space and speed (e.g., Cython) + # It then deploys to Layer's Beta and Prod account, including SAR Beta and Prod account. + # It uses canaries to attest Layers can be used and imported between stages. + # Lastly, it updates our documentation with the latest Layer ARN for all regions + # + # NOTE + # + # Watch out for the depth limit of 4 nested workflow_calls. + # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack + publish_layer: + needs: [seal, release, create_tag] + secrets: inherit + permissions: + id-token: write + contents: write + pages: write + pull-requests: write + uses: ./.github/workflows/publish_v2_layer.yml + with: + latest_published_version: ${{ needs.seal.outputs.RELEASE_VERSION }} + pre_release: ${{ inputs.pre_release }} + post_release: - needs: [build, release, publish_layer] + needs: [seal, release, publish_layer] permissions: contents: read issues: write @@ -194,7 +316,7 @@ jobs: pull-requests: write runs-on: ubuntu-latest env: - RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }} + RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - name: Close issues related to this release From 3da540a75d27fd04f013872cfd5b2c80727f6c8c Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Mon, 22 May 2023 14:42:50 +0200 Subject: [PATCH 3/9] chore: ruben's feedback Co-authored-by: Ruben Fonseca Signed-off-by: Heitor Lessa --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4be9c290b12..234098be7c1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,7 +111,7 @@ jobs: # # NOTE # - # we don't upload the artifact after tested to prevent no dependencies tamper our source code + # we don't upload the artifact after testing to prevent any tampering of our source code dependencies quality_check: needs: seal runs-on: ubuntu-latest From b0d4bb53d6241030178685eadda617b1f98cda73 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 22 May 2023 15:39:50 +0200 Subject: [PATCH 4/9] chore: document remaining sections; update release process doc --- .github/workflows/release.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 234098be7c1..598a1e8b2c1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,15 +4,17 @@ name: Release # # === Automated activities === # -# 1. Run tests, linting, security and complexity base line -# 2. Bump package version and build release artifact -# 3. Publish package to PyPi prod repository using cached artifact -# 4. Compile Layer and kick off pipeline for beta, prod, and canary releases -# 5. Update docs with latest Layer ARNs and Changelog -# 6. Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged -# 7. Builds a new user guide and API docs with release version; update /latest pointing to newly released version -# 8. Create PR to update package version on trunk -# 9. Close all issues labeled "pending-release" and notify customers about the release +# 1. [Seal] Bump to release version and export source code with integrity hash +# 2. [Quality check] Restore sealed source code, run tests, linting, security and complexity base line +# 3. [Build] Restore sealed source code, create and export hashed build artifact for PyPi release (wheel, tarball) +# 4. [Release] Restore built artifact, and publish package to PyPi prod repository +# 5. [Create Tag] Restore sealed source code, and create a new git tag using released version +# 6. [PR to bump version] Restore sealed source code, and create a PR to update trunk with latest released project metadata +# 7. [Publish Layer] Compile Layer and kick off pipeline for beta, prod, and canary releases +# 8. [Publish Layer] Update docs with latest Layer ARNs and Changelog +# 9. [Publish Layer] Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged +# 10. [Publish Layer] Builds a new user guide and API docs with release version; update /latest pointing to newly released version +# 11. [Post release] Close all issues labeled "pending-release" and notify customers about the release # # === Manual activities === # @@ -196,11 +198,12 @@ jobs: env: RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} steps: + # NOTE: we need actions/checkout in order to use our local actions (e.g., ./.github/actions) - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 with: ref: ${{ env.RELEASE_COMMIT }} - - name: Restore sealed source code + - name: Restore sealed built artifact uses: ./.github/actions/download-artifact with: name: release-${{ needs.seal.outputs.INTEGRITY_HASH }} From ef9d847d4c79c50524ad0042b56eeb72b4a42db8 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 22 May 2023 20:46:29 +0200 Subject: [PATCH 5/9] chore: include python bytecode in tarball for accurate hash verification --- .github/actions/upload-artifact/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/upload-artifact/action.yml b/.github/actions/upload-artifact/action.yml index ea6b4bc475a..ffa18cc0723 100644 --- a/.github/actions/upload-artifact/action.yml +++ b/.github/actions/upload-artifact/action.yml @@ -3,7 +3,7 @@ description: Wrapper around GitHub's official action, with additional archiving # PROCESS # -# 1. Creates tarball excluding .git and *.pyc files +# 1. Creates tarball excluding .git files # 2. Uploads tarball using actions/upload-artifact action, fail CI job if no file is found # 3. Remove archive after uploading it. @@ -61,7 +61,6 @@ runs: - name: Archive artifacts run: | tar --exclude-vcs \ - --exclude "*.pyc" \ -cvf "${ARCHIVE}" "${PATH_TO_ARCHIVE}" env: ARCHIVE: ${{ inputs.name }}.tar From 470a717f85a5876a64a816e793e23bd6f34660ab Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 22 May 2023 21:17:31 +0200 Subject: [PATCH 6/9] chore: add hash verification --- .github/workflows/release.yml | 69 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 598a1e8b2c1..f65b3e297bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,7 @@ jobs: permissions: contents: read outputs: - INTEGRITY_HASH: ${{ steps.integrity.outputs.INTEGRITY_HASH }} + SOURCE_CODE_HASH: ${{ steps.integrity.outputs.SOURCE_CODE_HASH }} RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }} steps: - name: Export release version @@ -88,7 +88,7 @@ jobs: - name: Create integrity hash id: integrity - run: echo "INTEGRITY_HASH=${SOURCE_CODE_HASH}" >> "$GITHUB_OUTPUT" + run: echo "SOURCE_CODE_HASH=${HASH}" >> "$GITHUB_OUTPUT" env: # paths to hash and why they're important to protect # @@ -99,12 +99,12 @@ jobs: # .github/ - github scripts and actions used in the release # docs/ - user guide documentation # examples/ - user guide code snippets - SOURCE_CODE_HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} - name: Upload sealed source code uses: ./.github/actions/upload-artifact with: - name: ${{ steps.integrity.outputs.INTEGRITY_HASH }} + name: ${{ steps.integrity.outputs.SOURCE_CODE_HASH }} path: . @@ -128,7 +128,7 @@ jobs: - name: Restore sealed source code uses: ./.github/actions/download-artifact with: - name: ${{ needs.seal.outputs.INTEGRITY_HASH }} + name: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} path: . - name: Debug cache restore @@ -156,6 +156,10 @@ jobs: needs: [quality_check, seal] permissions: contents: read + outputs: + BUILD_INTEGRITY_HASH: ${{ steps.integrity.outputs.BUILD_INTEGRITY_HASH }} + env: + SOURCE_INTEGRITY_HASH: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} steps: # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev) - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 @@ -165,7 +169,7 @@ jobs: - name: Restore sealed source code uses: ./.github/actions/download-artifact with: - name: ${{ needs.seal.outputs.INTEGRITY_HASH }} + name: ${{ env.SOURCE_INTEGRITY_HASH }} path: . - name: Install poetry @@ -179,10 +183,27 @@ jobs: - name: Build python package and wheel run: poetry build + # NOTE: Ran out of time to create a composite action out of this + # because GitHub Action inputs do not support arrays and it became fragile to join multiple strings then split + # keeping these hard coded for now until we have a cleaner way to reuse files/dirs we want to hash + - name: Source code tampering check + run: test "${SOURCE_INTEGRITY_HASH}" = "${CURRENT_HASH}" || exit 1 + env: + CURRENT_HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + + - name: Create integrity hash for build artifact + id: integrity + run: echo "BUILD_INTEGRITY_HASH=${HASH}" >> "$GITHUB_OUTPUT" + env: + # paths to hash and why they're important to protect + # + # dist/ - package distribution build + HASH: ${{ hashFiles('dist/**')}} + - name: Upload build artifact uses: ./.github/actions/upload-artifact with: - name: release-${{ needs.seal.outputs.INTEGRITY_HASH }} + name: build-${{ steps.integrity.outputs.INTEGRITY_HASH}} path: dist/ # This job uses release artifact to publish to PyPi @@ -197,18 +218,24 @@ jobs: id-token: write # OIDC for PyPi Trusted Publisher feature env: RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} + BUILD_INTEGRITY_HASH: ${{ needs.build.outputs.BUILD_INTEGRITY_HASH }} steps: # NOTE: we need actions/checkout in order to use our local actions (e.g., ./.github/actions) - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 with: ref: ${{ env.RELEASE_COMMIT }} - - name: Restore sealed built artifact + - name: Restore sealed build uses: ./.github/actions/download-artifact with: - name: release-${{ needs.seal.outputs.INTEGRITY_HASH }} + name: build-${{ env.BUILD_INTEGRITY_HASH }} path: . + - name: Source code tampering check + run: test "${BUILD_INTEGRITY_HASH}" = "${CURRENT_HASH}" || exit 1 + env: + CURRENT_HASH: ${{ hashFiles('dist/**')}} + - name: Upload to PyPi prod if: ${{ !inputs.skip_pypi }} uses: pypa/gh-action-pypi-publish@a56da0b891b3dc519c7ee3284aff1fad93cc8598 # v1.8.6 @@ -231,6 +258,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + env: + SOURCE_INTEGRITY_HASH: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} steps: # NOTE: we need actions/checkout to authenticate and configure git first - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 @@ -240,9 +269,17 @@ jobs: - name: Restore sealed source code uses: ./.github/actions/download-artifact with: - name: ${{ needs.seal.outputs.INTEGRITY_HASH }} + name: ${{ env.SOURCE_INTEGRITY_HASH }} path: . + # NOTE: Ran out of time to create a composite action out of this + # because GitHub Action inputs do not support arrays and it became fragile when making it reusable with strings + # keeping these hard coded for now until we have a cleaner way to reuse files/dirs we want to hash + - name: Source code tampering check + run: test "${SOURCE_INTEGRITY_HASH}" = "${CURRENT_HASH}" || exit 1 + env: + CURRENT_HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + - id: setup-git name: Git client setup and refresh tip run: | @@ -267,6 +304,8 @@ jobs: contents: write # create-pr action creates a temporary branch pull-requests: write # create-pr action creates a PR using the temporary branch runs-on: ubuntu-latest + env: + SOURCE_INTEGRITY_HASH: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} steps: # NOTE: we need actions/checkout to authenticate and configure git first - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 @@ -276,9 +315,17 @@ jobs: - name: Restore sealed source code uses: ./.github/actions/download-artifact with: - name: ${{ needs.seal.outputs.INTEGRITY_HASH }} + name: ${{ env.SOURCE_INTEGRITY_HASH }} path: . + # NOTE: Ran out of time to create a composite action out of this + # because GitHub Action inputs do not support arrays and it became fragile when making it reusable with strings + # keeping these hard coded for now until we have a cleaner way to reuse files/dirs we want to hash + - name: Source code tampering check + run: test "${SOURCE_INTEGRITY_HASH}" = "${CURRENT_HASH}" || exit 1 + env: + CURRENT_HASH: ${{ hashFiles('aws_lambda_powertools/**', 'pyproject.toml', 'poetry.lock', 'layer/**', '.github/**', 'docs/**', 'examples/**')}} + - name: Create PR id: create-pr uses: ./.github/actions/create-pr From df7bcb7a64e6529db7dc8674d738e44672305a6c Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 22 May 2023 21:22:04 +0200 Subject: [PATCH 7/9] chore: cleanup before review --- .github/workflows/release.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f65b3e297bc..8b177f5dda6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -104,7 +104,7 @@ jobs: - name: Upload sealed source code uses: ./.github/actions/upload-artifact with: - name: ${{ steps.integrity.outputs.SOURCE_CODE_HASH }} + name: source-${{ steps.integrity.outputs.SOURCE_CODE_HASH }} path: . @@ -119,6 +119,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + env: + SOURCE_INTEGRITY_HASH: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} steps: # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev) - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 @@ -128,7 +130,7 @@ jobs: - name: Restore sealed source code uses: ./.github/actions/download-artifact with: - name: ${{ needs.seal.outputs.SOURCE_CODE_HASH }} + name: source-${{ env.SOURCE_INTEGRITY_HASH }} path: . - name: Debug cache restore @@ -169,7 +171,7 @@ jobs: - name: Restore sealed source code uses: ./.github/actions/download-artifact with: - name: ${{ env.SOURCE_INTEGRITY_HASH }} + name: source-${{ env.SOURCE_INTEGRITY_HASH }} path: . - name: Install poetry @@ -269,7 +271,7 @@ jobs: - name: Restore sealed source code uses: ./.github/actions/download-artifact with: - name: ${{ env.SOURCE_INTEGRITY_HASH }} + name: source-${{ env.SOURCE_INTEGRITY_HASH }} path: . # NOTE: Ran out of time to create a composite action out of this @@ -315,7 +317,7 @@ jobs: - name: Restore sealed source code uses: ./.github/actions/download-artifact with: - name: ${{ env.SOURCE_INTEGRITY_HASH }} + name: source-${{ env.SOURCE_INTEGRITY_HASH }} path: . # NOTE: Ran out of time to create a composite action out of this @@ -369,6 +371,8 @@ jobs: RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + with: + ref: ${{ env.RELEASE_COMMIT }} - name: Close issues related to this release uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1 with: From 324190980f62a725ef51c44c4c41e00f5aefd2c4 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 23 May 2023 08:14:57 +0200 Subject: [PATCH 8/9] chore: fix build integrity hash reference --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8b177f5dda6..d5f22affe18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -205,7 +205,7 @@ jobs: - name: Upload build artifact uses: ./.github/actions/upload-artifact with: - name: build-${{ steps.integrity.outputs.INTEGRITY_HASH}} + name: build-${{ steps.integrity.outputs.BUILD_INTEGRITY_HASH}} path: dist/ # This job uses release artifact to publish to PyPi From e79287699a50b47c84144d69596ce3821a45ab5b Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 23 May 2023 09:48:59 +0200 Subject: [PATCH 9/9] chore: upgrade download-artifact to v3 due to node deprecation warnings --- .github/actions/download-artifact/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/download-artifact/action.yml b/.github/actions/download-artifact/action.yml index 85ae351c0e4..ef938ddb684 100644 --- a/.github/actions/download-artifact/action.yml +++ b/.github/actions/download-artifact/action.yml @@ -38,7 +38,7 @@ runs: using: composite steps: - name: Download artifacts - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # v2.1.1 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: ${{ inputs.name }} path: ${{ inputs.path }}