Skip to content

Commit 0f610dc

Browse files
leandrodamascenaleandrodamascena
authored
feat(lambda-layer): add pipeline to build Lambda layer in v3 (#4826)
* Pipeline for v3 * Pipeline for v3 * Pipeline for v3 * Adding v3 stack * Changing working directory * Addressing Andrea's feedback * Refactoring publish SAR * Bumping version * Renaming SAR name * Addressing Heitor's feedback * Addressing Heitor's feedback * Addressing Heitor's feedback * Addressing Heitor's feedback
1 parent 0d7c584 commit 0f610dc

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2789
-113
lines changed

Diff for: ‎.github/workflows/publish_v3_layer.yml

+314
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,314 @@
1+
name: Deploy v3 layer to all regions
2+
3+
# PROCESS
4+
#
5+
# 1. Compile Layer using cdk-aws-lambda-powertools-layer CDK construct for Python3.8-3.12 and x86/ARM architectures (uses custom runner as it's CPU heavy)
6+
# 2. Kick off pipeline for beta, prod, and canary releases
7+
# 3. Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged
8+
# 4. Builds and publishes docs with latest Layer ARN using given version (generally coming from release)
9+
10+
# USAGE
11+
#
12+
# NOTE: meant to be used with ./.github/workflows/release-v3.yml
13+
#
14+
# publish_layer:
15+
# needs: [seal, release, create_tag]
16+
# secrets: inherit
17+
# permissions:
18+
# id-token: write
19+
# contents: write
20+
# pages: write
21+
# pull-requests: write
22+
# uses: ./.github/workflows/publish_v2_layer.yml
23+
# with:
24+
# latest_published_version: ${{ needs.seal.outputs.RELEASE_VERSION }}
25+
# pre_release: ${{ inputs.pre_release }}
26+
# source_code_artifact_name: ${{ needs.seal.outputs.artifact_name }}
27+
# source_code_integrity_hash: ${{ needs.seal.outputs.integrity_hash }}
28+
29+
30+
on:
31+
workflow_dispatch:
32+
inputs:
33+
latest_published_version:
34+
description: "Latest PyPi published version to rebuild latest docs for, e.g. 3.0.0, 3.0.0a1 (pre-release)"
35+
required: true
36+
source_code_artifact_name:
37+
description: "Artifact name to restore sealed source code"
38+
type: string
39+
required: true
40+
source_code_integrity_hash:
41+
description: "Sealed source code integrity hash"
42+
type: string
43+
required: true
44+
pre_release:
45+
description: "Publishes documentation using a pre-release tag (3.0.0a1)."
46+
default: false
47+
type: boolean
48+
required: false
49+
workflow_call:
50+
inputs:
51+
latest_published_version:
52+
type: string
53+
description: "Latest PyPi published version to rebuild latest docs for, e.g. 3.0.0, 3.0.0a1 (pre-release)"
54+
required: true
55+
pre_release:
56+
description: "Publishes documentation using a pre-release tag (3.0.0a1)."
57+
default: false
58+
type: boolean
59+
required: false
60+
source_code_artifact_name:
61+
description: "Artifact name to restore sealed source code"
62+
type: string
63+
required: true
64+
source_code_integrity_hash:
65+
description: "Sealed source code integrity hash"
66+
type: string
67+
required: true
68+
69+
permissions:
70+
contents: read
71+
72+
73+
env:
74+
RELEASE_COMMIT: ${{ github.sha }}
75+
76+
jobs:
77+
build-layer:
78+
permissions:
79+
# lower privilege propagated from parent workflow (release.yml)
80+
contents: read
81+
id-token: write
82+
pages: none
83+
pull-requests: none
84+
runs-on: aws-powertools_ubuntu-latest_8-core
85+
strategy:
86+
max-parallel: 5
87+
matrix:
88+
python-version: ["3.8","3.9","3.10","3.11","3.12"]
89+
defaults:
90+
run:
91+
working-directory: ./layer_v3
92+
steps:
93+
- name: checkout
94+
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
95+
with:
96+
ref: ${{ env.RELEASE_COMMIT }}
97+
98+
- name: Restore sealed source code
99+
uses: ./.github/actions/seal-restore
100+
with:
101+
integrity_hash: ${{ inputs.source_code_integrity_hash }}
102+
artifact_name: ${{ inputs.source_code_artifact_name }}
103+
104+
- name: Install poetry
105+
run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0
106+
- name: Setup Node.js
107+
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
108+
with:
109+
node-version: "18.20.4"
110+
- name: Setup python
111+
uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
112+
with:
113+
python-version: ${{ matrix.python-version }}
114+
cache: "pip"
115+
- name: Resolve and install project dependencies
116+
# CDK spawns system python when compiling stack
117+
# therefore it ignores both activated virtual env and cached interpreter by GH
118+
run: |
119+
poetry export --format requirements.txt --output requirements.txt
120+
pip install --require-hashes -r requirements.txt
121+
122+
- name: Set up QEMU
123+
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v2.0.0
124+
with:
125+
platforms: arm64
126+
# NOTE: we need QEMU to build Layer against a different architecture (e.g., ARM)
127+
128+
- name: Set up Docker Buildx
129+
id: builder
130+
uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
131+
with:
132+
install: true
133+
driver: docker
134+
platforms: linux/amd64,linux/arm64
135+
136+
- name: Install CDK
137+
working-directory: ./
138+
run: |
139+
npm ci
140+
npx cdk --version
141+
142+
# Baking time for PyPi eventual consistency; 60s seemed more than enough
143+
# https://github.com/aws-powertools/powertools-lambda-python/issues/2491
144+
- name: Baking time (PyPi)
145+
run: sleep 60
146+
147+
- name: CDK build
148+
run: npx cdk synth --verbose --context version="${{ inputs.latest_published_version }}" --context pythonVersion="${{ matrix.python-version }}" -o cdk.out
149+
- name: zip output
150+
run: zip -r cdk.py${{ matrix.python-version }}.out.zip cdk.out
151+
- name: Archive CDK artifacts
152+
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
153+
with:
154+
name: cdk-layer-artifact-py${{ matrix.python-version }}
155+
path: layer/cdk.py${{ matrix.python-version }}.out.zip
156+
157+
beta:
158+
needs: build-layer
159+
# lower privilege propagated from parent workflow (release.yml)
160+
permissions:
161+
id-token: write
162+
contents: read
163+
pages: write # docs will be updated with latest Layer ARNs
164+
pull-requests: write # creation-action will create a PR with Layer ARN updates
165+
uses: ./.github/workflows/reusable_deploy_v3_layer_stack.yml
166+
secrets: inherit
167+
with:
168+
stage: "BETA"
169+
environment: "layer-beta"
170+
source_code_artifact_name: ${{ inputs.source_code_artifact_name }}
171+
source_code_integrity_hash: ${{ inputs.source_code_integrity_hash }}
172+
173+
prod:
174+
needs: beta
175+
# lower privilege propagated from parent workflow (release.yml)
176+
permissions:
177+
id-token: write
178+
contents: read
179+
pages: write # docs will be updated with latest Layer ARNs
180+
pull-requests: write # creation-action will create a PR with Layer ARN updates
181+
uses: ./.github/workflows/reusable_deploy_v3_layer_stack.yml
182+
secrets: inherit
183+
with:
184+
stage: "PROD"
185+
environment: "layer-prod"
186+
source_code_artifact_name: ${{ inputs.source_code_artifact_name }}
187+
source_code_integrity_hash: ${{ inputs.source_code_integrity_hash }}
188+
189+
sar-beta:
190+
needs: beta # canaries run on Layer Beta env
191+
permissions:
192+
# lower privilege propagated from parent workflow (release.yml)
193+
id-token: write
194+
contents: read
195+
pull-requests: none
196+
pages: none
197+
uses: ./.github/workflows/reusable_deploy_v3_sar.yml
198+
secrets: inherit
199+
with:
200+
stage: "BETA"
201+
environment: "layer-beta"
202+
package-version: ${{ inputs.latest_published_version }}
203+
source_code_artifact_name: ${{ inputs.source_code_artifact_name }}
204+
source_code_integrity_hash: ${{ inputs.source_code_integrity_hash }}
205+
206+
207+
sar-prod:
208+
needs: sar-beta
209+
permissions:
210+
# lower privilege propagated from parent workflow (release.yml)
211+
id-token: write
212+
contents: read
213+
pull-requests: none
214+
pages: none
215+
uses: ./.github/workflows/reusable_deploy_v3_sar.yml
216+
secrets: inherit
217+
with:
218+
stage: "PROD"
219+
environment: "layer-prod"
220+
package-version: ${{ inputs.latest_published_version }}
221+
source_code_artifact_name: ${{ inputs.source_code_artifact_name }}
222+
source_code_integrity_hash: ${{ inputs.source_code_integrity_hash }}
223+
224+
225+
# Updating the documentation with the latest Layer ARNs is a two-phase process
226+
#
227+
# 1. Update layer ARNs with latest deployed locally and create a PR with these changes
228+
# 2. Pull from temporary branch with these changes and update the docs we're releasing
229+
#
230+
# This keeps our permissions tight and we don't run into a conflict,
231+
# where a new release creates a new doc (2.16.0) while layers are still pointing to 2.15
232+
# because the PR has to be merged while release process is running
233+
234+
update_v3_layer_arn_docs:
235+
needs: prod
236+
outputs:
237+
temp_branch: ${{ steps.create-pr.outputs.temp_branch }}
238+
runs-on: ubuntu-latest
239+
permissions:
240+
# lower privilege propagated from parent workflow (release.yml)
241+
contents: write
242+
pull-requests: write
243+
id-token: none
244+
pages: none
245+
steps:
246+
- name: Checkout repository # reusable workflows start clean, so we need to checkout again
247+
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
248+
with:
249+
ref: ${{ env.RELEASE_COMMIT }}
250+
251+
- name: Restore sealed source code
252+
uses: ./.github/actions/seal-restore
253+
with:
254+
integrity_hash: ${{ inputs.source_code_integrity_hash }}
255+
artifact_name: ${{ inputs.source_code_artifact_name }}
256+
257+
- name: Download CDK layer artifacts
258+
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
259+
with:
260+
path: cdk-layer-stack
261+
pattern: cdk-layer-stack-* # merge all Layer artifacts created per region earlier (reusable_deploy_v2_layer_stack.yml; step "Save Layer ARN artifact")
262+
merge-multiple: true
263+
- name: Replace layer versions in documentation
264+
run: |
265+
ls -la cdk-layer-stack/
266+
./layer/scripts/update_layer_arn.sh cdk-layer-stack
267+
# NOTE: It felt unnecessary creating yet another PR to update changelog w/ latest tag
268+
# since this is the only step in the release where we update docs from a temp branch
269+
- name: Update changelog with latest tag
270+
run: make changelog
271+
- name: Create PR
272+
id: create-pr
273+
uses: ./.github/actions/create-pr
274+
with:
275+
files: "docs/index.md examples CHANGELOG.md"
276+
temp_branch_prefix: "ci-layer-docs"
277+
pull_request_title: "chore(ci): layer docs update"
278+
github_token: ${{ secrets.GITHUB_TOKEN }}
279+
280+
281+
prepare_docs_alias:
282+
runs-on: ubuntu-latest
283+
permissions:
284+
# lower privilege propagated from parent workflow (release.yml)
285+
contents: read
286+
pages: none
287+
id-token: none
288+
pull-requests: none
289+
outputs:
290+
DOCS_ALIAS: ${{ steps.set-alias.outputs.DOCS_ALIAS }}
291+
steps:
292+
- name: Set docs alias
293+
id: set-alias
294+
run: |
295+
DOCS_ALIAS=latest
296+
if [[ "${{ inputs.pre_release }}" == true ]] ; then
297+
DOCS_ALIAS=alpha
298+
fi
299+
echo DOCS_ALIAS="$DOCS_ALIAS" >> "$GITHUB_OUTPUT"
300+
301+
release_docs:
302+
needs: [update_v3_layer_arn_docs, prepare_docs_alias]
303+
permissions:
304+
# lower privilege propagated from parent workflow (release.yml)
305+
contents: write
306+
pages: write
307+
pull-requests: none
308+
id-token: write
309+
secrets: inherit
310+
uses: ./.github/workflows/reusable_publish_docs.yml
311+
with:
312+
version: ${{ inputs.latest_published_version }}
313+
alias: ${{ needs.prepare_docs_alias.outputs.DOCS_ALIAS }}
314+
git_ref: ${{ needs.update_v3_layer_arn_docs.outputs.temp_branch }}

0 commit comments

Comments
 (0)