From 567939b58c78bd381417ad1dd37a48fc59c14f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 10:32:21 +0200 Subject: [PATCH 01/45] feat(sar): add support for publishing SAR via Github Actions --- .github/workflows/publish_v2_layer.yml | 26 +++++++ .github/workflows/reusable_deploy_v2_sar.yml | 76 ++++++++++++++++++++ layer/sar/template.txt | 38 ++++++++++ 3 files changed, 140 insertions(+) create mode 100644 .github/workflows/reusable_deploy_v2_sar.yml create mode 100644 layer/sar/template.txt diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 469a94ad876..08d56d61ec4 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -23,6 +23,8 @@ jobs: defaults: run: working-directory: ./layer + outputs: + release-tag-version: ${{ steps.release-notes-tag.outputs.release-tag-version }} steps: - name: checkout uses: actions/checkout@v3 @@ -46,11 +48,13 @@ jobs: poetry export --format requirements.txt --output requirements.txt pip install -r requirements.txt - name: Set release notes tag + id: release-notes-tag run: | RELEASE_INPUT=${{ inputs.latest_published_version }} LATEST_TAG=$(git describe --tag --abbrev=0) RELEASE_TAG_VERSION=${RELEASE_INPUT:-$LATEST_TAG} echo RELEASE_TAG_VERSION="${RELEASE_TAG_VERSION:1}" >> "$GITHUB_ENV" + echo "::set-output name=release-tag-version::$RELEASE_TAG_VERSION" - name: Set up QEMU uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v2.0.0 # NOTE: we need QEMU to build Layer against a different architecture (e.g., ARM) @@ -81,6 +85,17 @@ jobs: artefact-name: "cdk-layer-artefact" environment: "layer-beta" + deploy-sar-beta: + needs: + - build-layer + uses: ./.github/workflows/reusable_deploy_v2_sar.yml + secrets: inherit + with: + stage: "BETA" + artefact-name: "cdk-layer-artefact" + environment: "layer-beta" + package-version: ${{ needs.build-layer.outputs.release-tag-version }} + # deploy-prod: # needs: # - deploy-beta @@ -90,3 +105,14 @@ jobs: # stage: "PROD" # artefact-name: "cdk-layer-artefact" # environment: "layer-prod" + + # deploy-sar-prod: + # needs: + # - build-layer + # uses: ./.github/workflows/reusable_deploy_v2_sar.yml + # secrets: inherit + # with: + # stage: "PROD" + # artefact-name: "cdk-layer-artefact" + # environment: "layer-beta" + # package-version: ${{ needs.build-layer.outputs.release-tag-version }} diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml new file mode 100644 index 00000000000..7e7883715a4 --- /dev/null +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -0,0 +1,76 @@ +name: Deploy V2 SAR + +permissions: + id-token: write + contents: read + +env: + SAM_VERSION: 2.44.0 + NODE_VERSION: 16.12 + AWS_REGION: eu-central-1 + +on: + workflow_call: + inputs: + stage: + description: "Deployment stage (BETA, PROD)" + required: true + type: string + artefact-name: + description: "CDK Layer Artefact name to download" + required: true + type: string + package-version: + description: "The version of the package to deploy" + required: true + type: string + environment: + description: "GitHub Environment to use for encrypted secrets" + required: true + type: string + +jobs: + deploy-cdk-stack: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + defaults: + run: + working-directory: ./layer + steps: + - name: checkout + uses: actions/checkout@v3 + - name: aws credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: $AWS_REGION + role-to-assume: ${{ secrets.AWS_SAR_ROLE_ARN }} + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: $NODE_VERSION + - name: install cdk and deps + run: | + npm install -g "aws-cdk@$CDK_VERSION" + cdk --version + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.artefact-name }} + path: layer + - name: unzip artefact + run: | + unzip cdk.out.zip + - name: Deploy x86_64 SAR + run: | + asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerStack.template.json) + sed -e "s//${{ inputs.package-version }}/g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s//.\/cdk.out\/$asset/g" sar/template.txt > sar/template.yml + cp ../README.md ../LICENSE "./cdk.out/$asset/" + pipx run sam=="$SAM_VERSION" package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} + pipx run sam=="$SAM_VERSION" publish --template packaged.yml --region "$AWS_REGION" + - name: Deploy arm64 SAR + run: | + asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerStack.template.json) + sed -e "s//${{ inputs.package-version }}/g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s//.\/cdk.out\/$asset/g" sar/template.txt > sar/template.yml + cp ../README.md ../LICENSE "./cdk.out/$asset/" + pipx run sam=="$SAM_VERSION" package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} + pipx run sam=="$SAM_VERSION" publish --template packaged.yml --region "$AWS_REGION" diff --git a/layer/sar/template.txt b/layer/sar/template.txt new file mode 100644 index 00000000000..8f8ea9085cf --- /dev/null +++ b/layer/sar/template.txt @@ -0,0 +1,38 @@ +AWSTemplateFormatVersion: '2010-09-09' + +Metadata: + AWS::ServerlessRepo::Application: + Name: + Description: "AWS Lambda Layer for aws-lambda-powertools " + Author: AWS + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE + ReadmeUrl: README.md + Labels: ['layer','lambda','powertools','python', 'aws'] + HomePageUrl: https://github.com/awslabs/aws-lambda-powertools-python + SemanticVersion: + SourceCodeUrl: https://github.com/awslabs/aws-lambda-powertools-python + +Transform: AWS::Serverless-2016-10-31 +Description: AWS Lambda Layer for aws-lambda-powertools with python 3.9, 3.8 or 3.7 + +Resources: + LambdaLayer: + Type: AWS::Serverless::LayerVersion + Properties: + Description: "AWS Lambda Layer for aws-lambda-powertools version " + LayerName: + ContentUri: + CompatibleRuntimes: + - python3.9 + - python3.8 + - python3.7 + LicenseInfo: 'Available under the Apache-2.0 license.' + RetentionPolicy: Retain + +Outputs: + LayerVersionArn: + Description: ARN for the published Layer version + Value: !Ref LambdaLayer + Export: + Name: !Sub 'LayerVersionArn-${AWS::StackName}' From 447f51a6f1d4e389c0a853a05fe4c1204c7fb8d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 11:39:18 +0200 Subject: [PATCH 02/45] chore: generate hashes for all deps --- layer/poetry.lock | 87 +++++++++++++++++++++++++++++--------------- layer/pyproject.toml | 1 - 2 files changed, 57 insertions(+), 31 deletions(-) diff --git a/layer/poetry.lock b/layer/poetry.lock index 6720fc5b96a..7e226bac519 100644 --- a/layer/poetry.lock +++ b/layer/poetry.lock @@ -7,10 +7,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "aws-cdk-lib" @@ -28,14 +28,14 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "boto3" -version = "1.24.88" +version = "1.24.89" description = "The AWS SDK for Python" category = "dev" optional = false python-versions = ">= 3.7" [package.dependencies] -botocore = ">=1.27.88,<1.28.0" +botocore = ">=1.27.89,<1.28.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -44,7 +44,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.27.88" +version = "1.27.89" description = "Low-level, data-driven core of boto 3." category = "dev" optional = false @@ -95,7 +95,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "constructs" -version = "10.1.124" +version = "10.1.128" description = "A programming model for software-defined state" category = "main" optional = false @@ -169,8 +169,8 @@ optional = false python-versions = ">=3.6" [package.extras] -testing = ["pytest-benchmark", "pytest"] -dev = ["tox", "pre-commit"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "publication" @@ -197,7 +197,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["railroad-diagrams", "jinja2"] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" @@ -269,8 +269,8 @@ optional = false python-versions = ">=3.5.3" [package.extras] -test = ["mypy", "typing-extensions", "pytest"] doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["mypy", "pytest", "typing-extensions"] [[package]] name = "typing-extensions" @@ -289,14 +289,14 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] -brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "cceb24edf99719274b946c811ad41ebbbdc0181593d4f07555c0977767cdd19a" +content-hash = "c2e19583ada823d6a5855df46e4851c1c21cd320fcb0edb57bd87e94b43e898c" [metadata.files] attrs = [ @@ -308,28 +308,37 @@ aws-cdk-lib = [ {file = "aws_cdk_lib-2.45.0-py3-none-any.whl", hash = "sha256:9463fe6d84563c4c23ae96810be0ea0ff0a260eebb85a4a7afe0c3747eca18a8"}, ] boto3 = [ - {file = "boto3-1.24.88-py3-none-any.whl", hash = "sha256:6b4cf1cd0be65202c4cf0e4c69099bac3620bcd4049ca25a5e223c668401dd69"}, - {file = "boto3-1.24.88.tar.gz", hash = "sha256:93934343cac76084600a520e5be70c52152364d0c410681c2e25c2290f0e151c"}, + {file = "boto3-1.24.89-py3-none-any.whl", hash = "sha256:346f8f0d101a4261dac146a959df18d024feda6431e1d9d84f94efd24d086cae"}, + {file = "boto3-1.24.89.tar.gz", hash = "sha256:d0d8ffcdc10821c4562bc7f935cdd840033bbc342ac0e14b6bdd348b3adf4c04"}, ] botocore = [ - {file = "botocore-1.27.88-py3-none-any.whl", hash = "sha256:de4e087b24cd3bc369eb2e27f8fe94a6499f7dea08c919fba13cefb2496bd2bb"}, - {file = "botocore-1.27.88.tar.gz", hash = "sha256:ded0a4035baf91eb358ef501c92a8512543f5ab7658f459df3077a70a555b5cd"}, + {file = "botocore-1.27.89-py3-none-any.whl", hash = "sha256:238f1dfdb8d8d017c2aea082609a3764f3161d32745900f41bcdcf290d95a048"}, + {file = "botocore-1.27.89.tar.gz", hash = "sha256:621f5413be8f97712b7e36c1b075a8791d1d1b9971a7ee060cdcdf5e2debf6c1"}, +] +cattrs = [ + {file = "cattrs-22.1.0-py3-none-any.whl", hash = "sha256:d55c477b4672f93606e992049f15d526dc7867e6c756cd6256d4af92e2b1e364"}, + {file = "cattrs-22.1.0.tar.gz", hash = "sha256:94b67b64cf92c994f8784c40c082177dc916e0489a73a9a36b24eb18a9db40c6"}, ] -cattrs = [] cdk-aws-lambda-powertools-layer = [ {file = "cdk-aws-lambda-powertools-layer-3.1.0.tar.gz", hash = "sha256:1e1457edbebfbb62ab2d21ae0f9b991db1b6c8508730fab8c99c3a7d3dfd99cb"}, {file = "cdk_aws_lambda_powertools_layer-3.1.0-py3-none-any.whl", hash = "sha256:c66cc722c924a50458418600df4060a1396e134df25b3cc85de59d5914541b31"}, ] -colorama = [] +colorama = [ + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, +] constructs = [ - {file = "constructs-10.1.124-py3-none-any.whl", hash = "sha256:76ef2e6776cfdd1c4131a18b0e83c1b154d462b3ebb37ddbeaf3043b3b0de301"}, - {file = "constructs-10.1.124.tar.gz", hash = "sha256:5e4bfcca275867e5cfb4636ef65ed334c861a816f287ce1136f082ad0e0c1c2c"}, + {file = "constructs-10.1.128-py3-none-any.whl", hash = "sha256:d6fbc88de4c2517b59e28a9d0bc3663e75decbe3464030b5bc53809868b52c9e"}, + {file = "constructs-10.1.128.tar.gz", hash = "sha256:6789412823ae27b39f659537337f688a9d555cad5845d4b821c7be02a061be1e"}, ] exceptiongroup = [ {file = "exceptiongroup-1.0.0rc9-py3-none-any.whl", hash = "sha256:2e3c3fc1538a094aab74fad52d6c33fc94de3dfee3ee01f187c0e0c72aec5337"}, {file = "exceptiongroup-1.0.0rc9.tar.gz", hash = "sha256:9086a4a21ef9b31c72181c77c040a074ba0889ee56a7b289ff0afb0d97655f96"}, ] -iniconfig = [] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] jmespath = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, @@ -338,16 +347,34 @@ jsii = [ {file = "jsii-1.69.0-py3-none-any.whl", hash = "sha256:f3ae5cdf5e854b4d59256dc1f8818cd3fabb8eb43fbd3134a8e8aef962643005"}, {file = "jsii-1.69.0.tar.gz", hash = "sha256:7c7ed2a913372add17d63322a640c6435324770eb78c6b89e4c701e07d9c84db"}, ] -packaging = [] -pluggy = [] -publication = [] -py = [] -pyparsing = [] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +publication = [ + {file = "publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6"}, + {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, +] +py = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] pytest = [ {file = "pytest-7.1.3-py3-none-any.whl", hash = "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7"}, {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, ] -python-dateutil = [] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] s3transfer = [ {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, diff --git a/layer/pyproject.toml b/layer/pyproject.toml index 4b0cada589f..13db130dcb9 100644 --- a/layer/pyproject.toml +++ b/layer/pyproject.toml @@ -8,7 +8,6 @@ license = "MIT" [tool.poetry.dependencies] python = "^3.9" cdk-aws-lambda-powertools-layer = "^3.1.0" -aws-cdk-lib = "^2.44.0" [tool.poetry.dev-dependencies] pytest = "^7.1.2" From 558e333f721d80c48c3631b4b9062c9d07313a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 13:37:52 +0200 Subject: [PATCH 03/45] chore: use git version to build layer --- .github/workflows/publish_v2_layer.yml | 5 ++++- layer/poetry.lock | 8 ++++---- layer/pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 08d56d61ec4..0c2dc2b00ad 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -52,6 +52,7 @@ jobs: run: | RELEASE_INPUT=${{ inputs.latest_published_version }} LATEST_TAG=$(git describe --tag --abbrev=0) + echo LATEST_TAG="${LATEST_TAG}" >> "$GITHUB_ENV" RELEASE_TAG_VERSION=${RELEASE_INPUT:-$LATEST_TAG} echo RELEASE_TAG_VERSION="${RELEASE_TAG_VERSION:1}" >> "$GITHUB_ENV" echo "::set-output name=release-tag-version::$RELEASE_TAG_VERSION" @@ -66,7 +67,9 @@ jobs: npm install -g aws-cdk@2.44.0 cdk --version - name: CDK build - run: cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out + run: | + # cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out + cdk synth --context version="git+https://github.com/awslabs/aws-lambda-powertools-python@$LATEST_TAG" -o cdk.out - name: zip output run: zip -r cdk.out.zip cdk.out - name: Archive CDK artifacts diff --git a/layer/poetry.lock b/layer/poetry.lock index 7e226bac519..d0fd4d0a37b 100644 --- a/layer/poetry.lock +++ b/layer/poetry.lock @@ -72,7 +72,7 @@ exceptiongroup = {version = "*", markers = "python_version <= \"3.10\""} [[package]] name = "cdk-aws-lambda-powertools-layer" -version = "3.1.0" +version = "3.2.0" description = "A lambda layer for AWS Powertools for python and typescript" category = "main" optional = false @@ -296,7 +296,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "c2e19583ada823d6a5855df46e4851c1c21cd320fcb0edb57bd87e94b43e898c" +content-hash = "10a4e60fe1abbe982f077699767b8a7949b2be5ca82f909647f34d1e30ffb9a9" [metadata.files] attrs = [ @@ -320,8 +320,8 @@ cattrs = [ {file = "cattrs-22.1.0.tar.gz", hash = "sha256:94b67b64cf92c994f8784c40c082177dc916e0489a73a9a36b24eb18a9db40c6"}, ] cdk-aws-lambda-powertools-layer = [ - {file = "cdk-aws-lambda-powertools-layer-3.1.0.tar.gz", hash = "sha256:1e1457edbebfbb62ab2d21ae0f9b991db1b6c8508730fab8c99c3a7d3dfd99cb"}, - {file = "cdk_aws_lambda_powertools_layer-3.1.0-py3-none-any.whl", hash = "sha256:c66cc722c924a50458418600df4060a1396e134df25b3cc85de59d5914541b31"}, + {file = "cdk-aws-lambda-powertools-layer-3.2.0.tar.gz", hash = "sha256:75b86a6c8714c82293d754f1d799134c4159953711312e261f8b3aaf77492fa6"}, + {file = "cdk_aws_lambda_powertools_layer-3.2.0-py3-none-any.whl", hash = "sha256:a293a2f42b459de70ccd9d2a16b0b0789f7c682aa31ab80d6696e93ff07caa92"}, ] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, diff --git a/layer/pyproject.toml b/layer/pyproject.toml index 13db130dcb9..e4caa660565 100644 --- a/layer/pyproject.toml +++ b/layer/pyproject.toml @@ -7,7 +7,7 @@ license = "MIT" [tool.poetry.dependencies] python = "^3.9" -cdk-aws-lambda-powertools-layer = "^3.1.0" +cdk-aws-lambda-powertools-layer = "^3.2.0" [tool.poetry.dev-dependencies] pytest = "^7.1.2" From 320f47fa9cd05e5b14e89645eb51b2f1190bf5d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 13:52:44 +0200 Subject: [PATCH 04/45] chore: fix action --- .github/workflows/reusable_deploy_v2_sar.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 7e7883715a4..36e291a8112 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -42,7 +42,7 @@ jobs: - name: aws credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-region: $AWS_REGION + aws-region: ${{ env.AWS_REGION }} role-to-assume: ${{ secrets.AWS_SAR_ROLE_ARN }} - name: Setup Node.js uses: actions/setup-node@v3 From 958e61c43c8f2ed64720db8c06b601d4183510f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 14:18:08 +0200 Subject: [PATCH 05/45] chore: rely on RELEASE_TAG_VERSION --- .github/workflows/publish_v2_layer.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 0c2dc2b00ad..3c7ffb0fcfc 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -52,7 +52,6 @@ jobs: run: | RELEASE_INPUT=${{ inputs.latest_published_version }} LATEST_TAG=$(git describe --tag --abbrev=0) - echo LATEST_TAG="${LATEST_TAG}" >> "$GITHUB_ENV" RELEASE_TAG_VERSION=${RELEASE_INPUT:-$LATEST_TAG} echo RELEASE_TAG_VERSION="${RELEASE_TAG_VERSION:1}" >> "$GITHUB_ENV" echo "::set-output name=release-tag-version::$RELEASE_TAG_VERSION" @@ -69,7 +68,7 @@ jobs: - name: CDK build run: | # cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out - cdk synth --context version="git+https://github.com/awslabs/aws-lambda-powertools-python@$LATEST_TAG" -o cdk.out + cdk synth --context version="git+https://github.com/awslabs/aws-lambda-powertools-python@RELEASE_TAG_VERSION" -o cdk.out - name: zip output run: zip -r cdk.out.zip cdk.out - name: Archive CDK artifacts From 1ddedf2e88fb4fcfdbf1beb9c5e55b645830f7db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 14:25:11 +0200 Subject: [PATCH 06/45] fix: typo --- .github/workflows/publish_v2_layer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 3c7ffb0fcfc..5eaae5fff19 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -68,7 +68,7 @@ jobs: - name: CDK build run: | # cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out - cdk synth --context version="git+https://github.com/awslabs/aws-lambda-powertools-python@RELEASE_TAG_VERSION" -o cdk.out + cdk synth --context version="git+https://github.com/awslabs/aws-lambda-powertools-python@$RELEASE_TAG_VERSION" -o cdk.out - name: zip output run: zip -r cdk.out.zip cdk.out - name: Archive CDK artifacts From 2132216c4b5609a20653c24c29a864662c3c152c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 14:30:11 +0200 Subject: [PATCH 07/45] fix: require version input --- .github/workflows/publish_v2_layer.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 5eaae5fff19..befc3850cad 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -8,8 +8,7 @@ on: workflow_dispatch: inputs: latest_published_version: - description: "Latest PyPi published version to rebuild latest docs for, e.g. v1.22.0" - default: "v2.0.0" + description: "Latest PyPi published version to rebuild latest docs for, e.g. v2.0.0" required: true # workflow_run: # workflows: ["Publish to PyPi"] From 470a09799e7b5bb3c6f7b9cf92f8aebd314898a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 14:33:40 +0200 Subject: [PATCH 08/45] chore: fix --- .github/workflows/publish_v2_layer.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index befc3850cad..3a1d74734c5 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -52,8 +52,8 @@ jobs: RELEASE_INPUT=${{ inputs.latest_published_version }} LATEST_TAG=$(git describe --tag --abbrev=0) RELEASE_TAG_VERSION=${RELEASE_INPUT:-$LATEST_TAG} - echo RELEASE_TAG_VERSION="${RELEASE_TAG_VERSION:1}" >> "$GITHUB_ENV" echo "::set-output name=release-tag-version::$RELEASE_TAG_VERSION" + echo RELEASE_TAG_VERSION="${RELEASE_TAG_VERSION:1}" >> "$GITHUB_ENV" - name: Set up QEMU uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v2.0.0 # NOTE: we need QEMU to build Layer against a different architecture (e.g., ARM) @@ -67,7 +67,7 @@ jobs: - name: CDK build run: | # cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out - cdk synth --context version="git+https://github.com/awslabs/aws-lambda-powertools-python@$RELEASE_TAG_VERSION" -o cdk.out + cdk synth --context version="git+https://github.com/awslabs/aws-lambda-powertools-python@${{ inputs.latest_published_version }}" -o cdk.out - name: zip output run: zip -r cdk.out.zip cdk.out - name: Archive CDK artifacts From 75a7a7c2374a624d97f7879b088cbf4130988a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 14:53:23 +0200 Subject: [PATCH 09/45] fix: rename stack names so it doesn't clash with v1 --- layer/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/layer/app.py b/layer/app.py index e44b05d453d..59a35dfd300 100644 --- a/layer/app.py +++ b/layer/app.py @@ -19,7 +19,7 @@ LayerStack( app, - "LayerStack", + "LayerV2Stack", powertools_version=POWERTOOLS_VERSION, ssm_paramter_layer_arn=SSM_PARAM_LAYER_ARN, ssm_parameter_layer_arm64_arn=SSM_PARAM_LAYER_ARM64_ARN, @@ -27,7 +27,7 @@ CanaryStack( app, - "CanaryStack", + "CanaryV2Stack", powertools_version=POWERTOOLS_VERSION, ssm_paramter_layer_arn=SSM_PARAM_LAYER_ARN, ssm_parameter_layer_arm64_arn=SSM_PARAM_LAYER_ARM64_ARN, From c9f64b993443bf15010fd1c16acd7da7cd7256f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Tue, 11 Oct 2022 16:04:40 +0200 Subject: [PATCH 10/45] fix: change stack names --- .github/workflows/reusable_deploy_v2_layer_stack.yml | 4 ++-- .github/workflows/reusable_deploy_v2_sar.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_layer_stack.yml b/.github/workflows/reusable_deploy_v2_layer_stack.yml index 8c4d45c7708..1a5eb83cd23 100644 --- a/.github/workflows/reusable_deploy_v2_layer_stack.yml +++ b/.github/workflows/reusable_deploy_v2_layer_stack.yml @@ -97,6 +97,6 @@ jobs: - name: unzip artefact run: unzip cdk.out.zip - name: CDK Deploy Layer - run: cdk deploy --app cdk.out --context region=${{ matrix.region }} 'LayerStack' --require-approval never --verbose + run: cdk deploy --app cdk.out --context region=${{ matrix.region }} 'LayerV2Stack' --require-approval never --verbose - name: CDK Deploy Canary - run: cdk deploy --app cdk.out --context region=${{ matrix.region}} --parameters DeployStage="${{ inputs.stage }}" 'CanaryStack' --require-approval never --verbose + run: cdk deploy --app cdk.out --context region=${{ matrix.region}} --parameters DeployStage="${{ inputs.stage }}" 'CanaryV2Stack' --require-approval never --verbose diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 36e291a8112..86baa05c07b 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -62,14 +62,14 @@ jobs: unzip cdk.out.zip - name: Deploy x86_64 SAR run: | - asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerStack.template.json) + asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s//${{ inputs.package-version }}/g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s//.\/cdk.out\/$asset/g" sar/template.txt > sar/template.yml cp ../README.md ../LICENSE "./cdk.out/$asset/" pipx run sam=="$SAM_VERSION" package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} pipx run sam=="$SAM_VERSION" publish --template packaged.yml --region "$AWS_REGION" - name: Deploy arm64 SAR run: | - asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerStack.template.json) + asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s//${{ inputs.package-version }}/g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s//.\/cdk.out\/$asset/g" sar/template.txt > sar/template.yml cp ../README.md ../LICENSE "./cdk.out/$asset/" pipx run sam=="$SAM_VERSION" package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} From 6c9c066aaf8cd71336650c28c622deee953d0646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 10:45:55 +0200 Subject: [PATCH 11/45] fix: use new SAR role --- .github/workflows/publish_v2_layer.yml | 2 +- .github/workflows/reusable_deploy_v2_sar.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 3a1d74734c5..ef700774c8b 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -115,5 +115,5 @@ jobs: # with: # stage: "PROD" # artefact-name: "cdk-layer-artefact" - # environment: "layer-beta" + # environment: "layer-prod" # package-version: ${{ needs.build-layer.outputs.release-tag-version }} diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 86baa05c07b..5efe1db3b60 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -43,7 +43,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_SAR_ROLE_ARN }} + role-to-assume: ${{ secrets.AWS_SAR_V2_ROLE_ARN }} - name: Setup Node.js uses: actions/setup-node@v3 with: From 17b6f3da0c5de61ad2e0f2d70ccdc90b0a318fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 11:23:46 +0200 Subject: [PATCH 12/45] fix: assume role after assuming OIDC role --- .github/workflows/reusable_deploy_v2_sar.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 5efe1db3b60..9690c99df91 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -43,7 +43,12 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_SAR_V2_ROLE_ARN }} + role-to-assume: ${{ secrets.AWS_LAYERS_ROLE_ARN }} + - name: aws credentials SAR role + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_AWS_SAR_V2_ROLE_ARN }} - name: Setup Node.js uses: actions/setup-node@v3 with: From 09b984d2a7870e81c85d484fc26e383002df968a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 11:35:39 +0200 Subject: [PATCH 13/45] fix: typo --- .github/workflows/reusable_deploy_v2_sar.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 9690c99df91..b6fe32d9406 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -48,7 +48,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ secrets.AWS_AWS_SAR_V2_ROLE_ARN }} + role-to-assume: ${{ secrets.AWS_SAR_V2_ROLE_ARN }} - name: Setup Node.js uses: actions/setup-node@v3 with: From 498c6c3899a5515580d4ae8b3de513015c23865f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 13:30:10 +0200 Subject: [PATCH 14/45] fix: assume sar --- .github/workflows/reusable_deploy_v2_sar.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index b6fe32d9406..6cef3ca67a1 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -47,6 +47,9 @@ jobs: - name: aws credentials SAR role uses: aws-actions/configure-aws-credentials@v1 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} aws-region: ${{ env.AWS_REGION }} role-to-assume: ${{ secrets.AWS_SAR_V2_ROLE_ARN }} - name: Setup Node.js From 7f1fe802546ad20f89ed0f627f7faf621c656ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 14:44:13 +0200 Subject: [PATCH 15/45] fix: role duration session --- .github/workflows/reusable_deploy_v2_sar.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 6cef3ca67a1..c004a886870 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -50,6 +50,7 @@ jobs: aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-duration-seconds: 1200 aws-region: ${{ env.AWS_REGION }} role-to-assume: ${{ secrets.AWS_SAR_V2_ROLE_ARN }} - name: Setup Node.js From a3eecbf2aebd0690500ae4c00e4f5cdf386d2f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 14:50:06 +0200 Subject: [PATCH 16/45] fix: env --- .github/workflows/reusable_deploy_v2_sar.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index c004a886870..4b779abf584 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -56,7 +56,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v3 with: - node-version: $NODE_VERSION + node-version: ${{ env.NODE_VERSION }} - name: install cdk and deps run: | npm install -g "aws-cdk@$CDK_VERSION" From 91bc1e1f378987178ce80444db4822657848892a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 15:01:35 +0200 Subject: [PATCH 17/45] fix: sed escape --- .github/workflows/reusable_deploy_v2_sar.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 4b779abf584..dea1d21d9c4 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -72,14 +72,14 @@ jobs: - name: Deploy x86_64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) - sed -e "s//${{ inputs.package-version }}/g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s//.\/cdk.out\/$asset/g" sar/template.txt > sar/template.yml + sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml cp ../README.md ../LICENSE "./cdk.out/$asset/" pipx run sam=="$SAM_VERSION" package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} pipx run sam=="$SAM_VERSION" publish --template packaged.yml --region "$AWS_REGION" - name: Deploy arm64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) - sed -e "s//${{ inputs.package-version }}/g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s//.\/cdk.out\/$asset/g" sar/template.txt > sar/template.yml + sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml cp ../README.md ../LICENSE "./cdk.out/$asset/" pipx run sam=="$SAM_VERSION" package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} pipx run sam=="$SAM_VERSION" publish --template packaged.yml --region "$AWS_REGION" From d50d531f6a11e89ee1ff5bb0434f15282d405003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 15:29:29 +0200 Subject: [PATCH 18/45] fix: sam version --- .github/workflows/reusable_deploy_v2_sar.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index dea1d21d9c4..e82d39fc25b 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -74,12 +74,12 @@ jobs: asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml cp ../README.md ../LICENSE "./cdk.out/$asset/" - pipx run sam=="$SAM_VERSION" package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} - pipx run sam=="$SAM_VERSION" publish --template packaged.yml --region "$AWS_REGION" + sam package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} + sam publish --template packaged.yml --region "$AWS_REGION" - name: Deploy arm64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml cp ../README.md ../LICENSE "./cdk.out/$asset/" - pipx run sam=="$SAM_VERSION" package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} - pipx run sam=="$SAM_VERSION" publish --template packaged.yml --region "$AWS_REGION" + sam package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} + sam publish --template packaged.yml --region "$AWS_REGION" From c467ee04a0b08cb54adeba48bd55b52cd4f0d26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 15:56:59 +0200 Subject: [PATCH 19/45] fix: remove cdk --- .github/workflows/reusable_deploy_v2_sar.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index e82d39fc25b..354441b8fed 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -57,10 +57,6 @@ jobs: uses: actions/setup-node@v3 with: node-version: ${{ env.NODE_VERSION }} - - name: install cdk and deps - run: | - npm install -g "aws-cdk@$CDK_VERSION" - cdk --version - name: Download artifact uses: actions/download-artifact@v3 with: @@ -73,13 +69,13 @@ jobs: run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml - cp ../README.md ../LICENSE "./cdk.out/$asset/" + cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" - name: Deploy arm64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml - cp ../README.md ../LICENSE "./cdk.out/$asset/" + cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" From ede9983e3193b77ab97cf2c5c5638d82409e9c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 16:06:54 +0200 Subject: [PATCH 20/45] fix: debug --- .github/workflows/reusable_deploy_v2_sar.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 354441b8fed..4b562938a51 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -69,13 +69,14 @@ jobs: run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml - cp README.md LICENSE "./cdk.out/$asset/" + cp ../README.md ../LICENSE "./cdk.out/$asset/" + ls -la "./cdk.out/$asset/" sam package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" - name: Deploy arm64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml - cp README.md LICENSE "./cdk.out/$asset/" + cp ../README.md ../LICENSE "./cdk.out/$asset/" sam package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" From 99b88a3ad04376cb38de1dfa7f05e5d4523b48b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 16:23:46 +0200 Subject: [PATCH 21/45] fix: readme and license path --- layer/sar/template.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/layer/sar/template.txt b/layer/sar/template.txt index 8f8ea9085cf..808e9d7a36a 100644 --- a/layer/sar/template.txt +++ b/layer/sar/template.txt @@ -6,8 +6,8 @@ Metadata: Description: "AWS Lambda Layer for aws-lambda-powertools " Author: AWS SpdxLicenseId: Apache-2.0 - LicenseUrl: LICENSE - ReadmeUrl: README.md + LicenseUrl: /LICENSE + ReadmeUrl: /README.md Labels: ['layer','lambda','powertools','python', 'aws'] HomePageUrl: https://github.com/awslabs/aws-lambda-powertools-python SemanticVersion: From ae235bdfd154eb292dd964e0b6cb9027d0e70a57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 16:40:09 +0200 Subject: [PATCH 22/45] fix: path --- .github/workflows/reusable_deploy_v2_sar.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 4b562938a51..2783583002d 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -68,15 +68,15 @@ jobs: - name: Deploy x86_64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) - sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml + sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s||./cdk.out/$asset|g" sar/template.txt > template.yml cp ../README.md ../LICENSE "./cdk.out/$asset/" ls -la "./cdk.out/$asset/" - sam package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} + sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" - name: Deploy arm64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) - sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s||./cdk.out/$asset|g" sar/template.txt > sar/template.yml + sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s||./cdk.out/$asset|g" sar/template.txt > template.yml cp ../README.md ../LICENSE "./cdk.out/$asset/" - sam package --template-file sar/template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} + sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" From 477545de77dbd38f582d54fce04462cc9881e668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 16:48:05 +0200 Subject: [PATCH 23/45] fix: change SAR region --- .github/workflows/reusable_deploy_v2_sar.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 2783583002d..5d8869bfcd9 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -7,7 +7,7 @@ permissions: env: SAM_VERSION: 2.44.0 NODE_VERSION: 16.12 - AWS_REGION: eu-central-1 + AWS_REGION: eu-west-1 on: workflow_call: From 45e8c27d0443b4e25f51ef8b658b0100ff4ab1c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 20:46:25 +0200 Subject: [PATCH 24/45] fix: force version --- .github/workflows/publish_v2_layer.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index ef700774c8b..49ff915675a 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -95,7 +95,8 @@ jobs: stage: "BETA" artefact-name: "cdk-layer-artefact" environment: "layer-beta" - package-version: ${{ needs.build-layer.outputs.release-tag-version }} + package-version: 2.0.0 + # package-version: ${{ needs.build-layer.outputs.release-tag-version }} # deploy-prod: # needs: From 66ca24aaeca8386c033a949b8f7f840ca97787a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 21:20:59 +0200 Subject: [PATCH 25/45] chore: now remove the test SAR --- .github/workflows/reusable_deploy_v2_sar.yml | 40 +++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 5d8869bfcd9..4119f464095 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -8,6 +8,7 @@ env: SAM_VERSION: 2.44.0 NODE_VERSION: 16.12 AWS_REGION: eu-west-1 + SAR_NAME: aws-lambda-powertools-python-layer-v2 on: workflow_call: @@ -30,12 +31,9 @@ on: type: string jobs: - deploy-cdk-stack: + deploy-sar-app: runs-on: ubuntu-latest environment: ${{ inputs.environment }} - defaults: - run: - working-directory: ./layer steps: - name: checkout uses: actions/checkout@v3 @@ -46,6 +44,7 @@ jobs: role-to-assume: ${{ secrets.AWS_LAYERS_ROLE_ARN }} - name: aws credentials SAR role uses: aws-actions/configure-aws-credentials@v1 + id: aws-credentials-sar-role with: aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} @@ -65,18 +64,41 @@ jobs: - name: unzip artefact run: | unzip cdk.out.zip + - name: Set SAR name + run: | + name=${{ env.SAR_NAME }} + if [[ "${{ inputs.stage }}" == "BETA" ]]; then + name=test-$sar_name + fi + echo SAR__NAME="${name}" >> "$GITHUB_ENV" - name: Deploy x86_64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) - sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2/g" -e "s||./cdk.out/$asset|g" sar/template.txt > template.yml - cp ../README.md ../LICENSE "./cdk.out/$asset/" - ls -la "./cdk.out/$asset/" + sed -e "s||${{ inputs.package-version }}|g" -e "s//${{ env.SAR_NAME }}/g" -e "s||./cdk.out/$asset|g" layer/sar/template.txt > template.yml + cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" - name: Deploy arm64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) - sed -e "s||${{ inputs.package-version }}|g" -e "s//aws-lambda-powertools-python-layer-v2-arm64/g" -e "s||./cdk.out/$asset|g" sar/template.txt > template.yml - cp ../README.md ../LICENSE "./cdk.out/$asset/" + sed -e "s||${{ inputs.package-version }}|g" -e "s//${{ env.SAR_NAME }}-arm64/g" -e "s||./cdk.out/$asset|g" layer/sar/template.txt > template.yml + cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" + # - name: Deploy canary + - name: Cleanup BETA x86_64 SAR + if: ${{ inputs.stage == 'BETA' }} + run: | + app_id=$(aws serverlessrepo list-applications --output text --query "Applications[?ApplicationId=='arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }}'].ApplicationId") + if [ -z "${app_id}" ]; then + echo "Could not find the SAR that we just deployed." + fi + aws serverlessrepo delete-application --application-id "$app_id" + - name: Cleanup BETA arm64 SAR + if: ${{ inputs.stage == 'BETA' }} + run: | + app_id=$(aws serverlessrepo list-applications --output text --query "Applications[?ApplicationId=='arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }}-arm64'].ApplicationId") + if [ -z "${app_id}" ]; then + echo "Could not find the SAR that we just deployed." + fi + aws serverlessrepo delete-application --application-id "$app_id" From 46435deef37045506c2b992408f03219717f11be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 21:28:49 +0200 Subject: [PATCH 26/45] fix: artifact path --- .github/workflows/reusable_deploy_v2_sar.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 4119f464095..2e490c665b4 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -60,7 +60,6 @@ jobs: uses: actions/download-artifact@v3 with: name: ${{ inputs.artefact-name }} - path: layer - name: unzip artefact run: | unzip cdk.out.zip From 90c175b45f513914985ebfcb48dab8dcca6f268a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 21:44:46 +0200 Subject: [PATCH 27/45] feat: add SAR canary --- .github/workflows/reusable_deploy_v2_sar.yml | 40 +++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 2e490c665b4..3d55ae800ad 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -35,14 +35,14 @@ jobs: runs-on: ubuntu-latest environment: ${{ inputs.environment }} steps: - - name: checkout + - name: Checkout uses: actions/checkout@v3 - - name: aws credentials + - name: AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-region: ${{ env.AWS_REGION }} role-to-assume: ${{ secrets.AWS_LAYERS_ROLE_ARN }} - - name: aws credentials SAR role + - name: AWS credentials SAR role uses: aws-actions/configure-aws-credentials@v1 id: aws-credentials-sar-role with: @@ -60,7 +60,7 @@ jobs: uses: actions/download-artifact@v3 with: name: ${{ inputs.artefact-name }} - - name: unzip artefact + - name: Unzip artefact run: | unzip cdk.out.zip - name: Set SAR name @@ -69,7 +69,7 @@ jobs: if [[ "${{ inputs.stage }}" == "BETA" ]]; then name=test-$sar_name fi - echo SAR__NAME="${name}" >> "$GITHUB_ENV" + echo SAR_NAME="${name}" >> "$GITHUB_ENV" - name: Deploy x86_64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) @@ -84,7 +84,35 @@ jobs: cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" - # - name: Deploy canary + - name: Deploy BETA canary + if: ${{ inputs.stage == 'BETA' }} + run: | + STACK_NAME=serverlessrepo-v2-testing + + echo "Check if stack does not exist" + stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName > '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}") + echo "$stack_exists" + if [[ -n "$stack_exists" ]] ; then + echo "Found test deployment stack, removing..." + aws cloudformation delete-stack --stack-name "$STACK_NAME" + aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" + fi + + echo "Creating canary stack" + aws serverlessrepo create-cloud-formation-change-set --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --stack-name "$STACK_NAME" --capabilities CAPABILITY_NAMED_IAM + CHANGE_SET_ID=$(aws cloudformation list-change-sets --stack-name "$STACK_NAME" --query 'Summaries[*].ChangeSetId' --output text) + aws cloudformation wait change-set-create-complete --change-set-name "$CHANGE_SET_ID" + aws cloudformation execute-change-set --change-set-name "$CHANGE_SET_ID" + aws cloudformation wait stack-create-complete --stack-name "$STACK_NAME" + echo "Waiting until stack deployment completes..." + + echo "Exit with error if stack is not in CREATE_COMPLETE" + stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName > '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}") + if [[ -z "$stack_exists" ]] ; then + echo "Could find successful deployment, exit error..." + exit 1 + fi + echo "Deployment successful" - name: Cleanup BETA x86_64 SAR if: ${{ inputs.stage == 'BETA' }} run: | From 25dd302469fc0159af9e7b53e1a019eba5e4322e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Wed, 12 Oct 2022 21:53:06 +0200 Subject: [PATCH 28/45] fix: typo --- .github/workflows/reusable_deploy_v2_sar.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 3d55ae800ad..6e008b1e765 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -65,11 +65,11 @@ jobs: unzip cdk.out.zip - name: Set SAR name run: | - name=${{ env.SAR_NAME }} + NAME=${{ env.SAR_NAME }} if [[ "${{ inputs.stage }}" == "BETA" ]]; then - name=test-$sar_name + NAME="test-$NAME" fi - echo SAR_NAME="${name}" >> "$GITHUB_ENV" + echo SAR_NAME="${NAME}" >> "$GITHUB_ENV" - name: Deploy x86_64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) From 4a666429f4e3c88468fc8e0669f2432ee669fd66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 07:57:22 +0200 Subject: [PATCH 29/45] fix: aws output --- .github/workflows/reusable_deploy_v2_sar.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 6e008b1e765..a4269058c35 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -87,10 +87,10 @@ jobs: - name: Deploy BETA canary if: ${{ inputs.stage == 'BETA' }} run: | - STACK_NAME=serverlessrepo-v2-testing + STACK_NAME=serverlessrepo-v2-powertools-layer-test-stack echo "Check if stack does not exist" - stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName > '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}") + stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName > '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}" --output text) echo "$stack_exists" if [[ -n "$stack_exists" ]] ; then echo "Found test deployment stack, removing..." From e0774a7cd376bb26f64208d3186ce0bbbbb68b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 08:17:39 +0200 Subject: [PATCH 30/45] fix: debug --- .github/workflows/reusable_deploy_v2_sar.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index a4269058c35..e4cb7fcab9f 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -99,6 +99,7 @@ jobs: fi echo "Creating canary stack" + echo "Stack name: $STACK_NAME" aws serverlessrepo create-cloud-formation-change-set --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --stack-name "$STACK_NAME" --capabilities CAPABILITY_NAMED_IAM CHANGE_SET_ID=$(aws cloudformation list-change-sets --stack-name "$STACK_NAME" --query 'Summaries[*].ChangeSetId' --output text) aws cloudformation wait change-set-create-complete --change-set-name "$CHANGE_SET_ID" From 46425eab54b9d5bcc707caee0243a6211bfd8f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 09:45:03 +0200 Subject: [PATCH 31/45] fix: it will work now --- .github/workflows/reusable_deploy_v2_sar.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index e4cb7fcab9f..371176af8af 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -90,7 +90,7 @@ jobs: STACK_NAME=serverlessrepo-v2-powertools-layer-test-stack echo "Check if stack does not exist" - stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName > '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}" --output text) + stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}" --output text) echo "$stack_exists" if [[ -n "$stack_exists" ]] ; then echo "Found test deployment stack, removing..." @@ -100,7 +100,7 @@ jobs: echo "Creating canary stack" echo "Stack name: $STACK_NAME" - aws serverlessrepo create-cloud-formation-change-set --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --stack-name "$STACK_NAME" --capabilities CAPABILITY_NAMED_IAM + aws serverlessrepo create-cloud-formation-change-set --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --stack-name "${STACK_NAME/serverlessrepo-//}" --capabilities CAPABILITY_NAMED_IAM CHANGE_SET_ID=$(aws cloudformation list-change-sets --stack-name "$STACK_NAME" --query 'Summaries[*].ChangeSetId' --output text) aws cloudformation wait change-set-create-complete --change-set-name "$CHANGE_SET_ID" aws cloudformation execute-change-set --change-set-name "$CHANGE_SET_ID" From e7fa0ae89cf77d89b142b0551eead693a626f1ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 09:55:36 +0200 Subject: [PATCH 32/45] fix: typo --- .github/workflows/reusable_deploy_v2_sar.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 371176af8af..2ef038589b0 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -100,7 +100,7 @@ jobs: echo "Creating canary stack" echo "Stack name: $STACK_NAME" - aws serverlessrepo create-cloud-formation-change-set --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --stack-name "${STACK_NAME/serverlessrepo-//}" --capabilities CAPABILITY_NAMED_IAM + aws serverlessrepo create-cloud-formation-change-set --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --stack-name "${STACK_NAME/serverlessrepo-/}" --capabilities CAPABILITY_NAMED_IAM CHANGE_SET_ID=$(aws cloudformation list-change-sets --stack-name "$STACK_NAME" --query 'Summaries[*].ChangeSetId' --output text) aws cloudformation wait change-set-create-complete --change-set-name "$CHANGE_SET_ID" aws cloudformation execute-change-set --change-set-name "$CHANGE_SET_ID" @@ -108,7 +108,7 @@ jobs: echo "Waiting until stack deployment completes..." echo "Exit with error if stack is not in CREATE_COMPLETE" - stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName > '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}") + stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}") if [[ -z "$stack_exists" ]] ; then echo "Could find successful deployment, exit error..." exit 1 From 226242f536e8040cc9db7a5727f7685843e1b2d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 10:23:54 +0200 Subject: [PATCH 33/45] chore: add production --- .github/workflows/publish_v2_layer.yml | 12 ++++++++++++ .github/workflows/reusable_deploy_v2_sar.yml | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 49ff915675a..c68e75af264 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -98,6 +98,18 @@ jobs: package-version: 2.0.0 # package-version: ${{ needs.build-layer.outputs.release-tag-version }} + deploy-sar-prod: + needs: + - deploy-sar-beta + uses: ./.github/workflows/reusable_deploy_v2_sar.yml + secrets: inherit + with: + stage: "PROD" + artefact-name: "cdk-layer-artefact" + environment: "layer-prod" + package-version: 2.0.0 + # package-version: ${{ needs.build-layer.outputs.release-tag-version }} + # deploy-prod: # needs: # - deploy-beta diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 2ef038589b0..1cbd5099e06 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -63,7 +63,7 @@ jobs: - name: Unzip artefact run: | unzip cdk.out.zip - - name: Set SAR name + - name: Set test SAR name run: | NAME=${{ env.SAR_NAME }} if [[ "${{ inputs.stage }}" == "BETA" ]]; then @@ -77,6 +77,13 @@ jobs: cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" + - name: Publish x86_64 SAR + if: ${{ inputs.stage == 'PROD' }} + run: | + # wait until sar registers the app, otherwise it fails to make it public + sleep 15 + echo "Make x86_64 SAR app public" + aws serverlessrepo put-application-policy --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --statements Principals='*',Actions=Deploy - name: Deploy arm64 SAR run: | asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) @@ -84,6 +91,13 @@ jobs: cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" + - name: Publish arm64 SAR + if: ${{ inputs.stage == 'PROD' }} + run: | + # wait until sar registers the app, otherwise it fails to make it public + sleep 15 + echo "Make arm64 SAR app public" + aws serverlessrepo put-application-policy --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }}-arm64 --statements Principals='*',Actions=Deploy - name: Deploy BETA canary if: ${{ inputs.stage == 'BETA' }} run: | From 9d6177c2b4018a41a7fbac458a216a2f51166659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 11:07:38 +0200 Subject: [PATCH 34/45] chore: parallelize SAR --- .github/workflows/reusable_deploy_v2_sar.yml | 44 +++++++------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 1cbd5099e06..815420e8cea 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -34,6 +34,9 @@ jobs: deploy-sar-app: runs-on: ubuntu-latest environment: ${{ inputs.environment }} + strategy: + matrix: + architecture: ["x86_64", "arm64"] steps: - name: Checkout uses: actions/checkout@v3 @@ -63,46 +66,39 @@ jobs: - name: Unzip artefact run: | unzip cdk.out.zip - - name: Set test SAR name + - name: Set SAR name run: | NAME=${{ env.SAR_NAME }} if [[ "${{ inputs.stage }}" == "BETA" ]]; then NAME="test-$NAME" fi echo SAR_NAME="${NAME}" >> "$GITHUB_ENV" - - name: Deploy x86_64 SAR + - name: Adds arm64 suffix to SAR name + if: ${{ matrix.architecture == 'arm64' }} + run: echo SAR_NAME="${SAR_NAME}-arm64" >> "$GITHUB_ENV" + - name: Deploy SAR run: | - asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["x86_64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) + asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["${{ matrix.architecture }}"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) sed -e "s||${{ inputs.package-version }}|g" -e "s//${{ env.SAR_NAME }}/g" -e "s||./cdk.out/$asset|g" layer/sar/template.txt > template.yml cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" - - name: Publish x86_64 SAR + - name: Publish SAR if: ${{ inputs.stage == 'PROD' }} run: | # wait until sar registers the app, otherwise it fails to make it public sleep 15 - echo "Make x86_64 SAR app public" + echo "Make SAR app public" aws serverlessrepo put-application-policy --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --statements Principals='*',Actions=Deploy - - name: Deploy arm64 SAR - run: | - asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["arm64"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) - sed -e "s||${{ inputs.package-version }}|g" -e "s//${{ env.SAR_NAME }}-arm64/g" -e "s||./cdk.out/$asset|g" layer/sar/template.txt > template.yml - cp README.md LICENSE "./cdk.out/$asset/" - sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} - sam publish --template packaged.yml --region "$AWS_REGION" - - name: Publish arm64 SAR - if: ${{ inputs.stage == 'PROD' }} - run: | - # wait until sar registers the app, otherwise it fails to make it public - sleep 15 - echo "Make arm64 SAR app public" - aws serverlessrepo put-application-policy --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }}-arm64 --statements Principals='*',Actions=Deploy - name: Deploy BETA canary if: ${{ inputs.stage == 'BETA' }} run: | STACK_NAME=serverlessrepo-v2-powertools-layer-test-stack + if [[ "${{ matrix.architecture }}" == "arm64" ]]; then + STACK_NAME="${STACK_NAME}-arm64" + fi + echo "Check if stack does not exist" stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}" --output text) echo "$stack_exists" @@ -128,7 +124,7 @@ jobs: exit 1 fi echo "Deployment successful" - - name: Cleanup BETA x86_64 SAR + - name: Cleanup BETA SAR if: ${{ inputs.stage == 'BETA' }} run: | app_id=$(aws serverlessrepo list-applications --output text --query "Applications[?ApplicationId=='arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }}'].ApplicationId") @@ -136,11 +132,3 @@ jobs: echo "Could not find the SAR that we just deployed." fi aws serverlessrepo delete-application --application-id "$app_id" - - name: Cleanup BETA arm64 SAR - if: ${{ inputs.stage == 'BETA' }} - run: | - app_id=$(aws serverlessrepo list-applications --output text --query "Applications[?ApplicationId=='arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }}-arm64'].ApplicationId") - if [ -z "${app_id}" ]; then - echo "Could not find the SAR that we just deployed." - fi - aws serverlessrepo delete-application --application-id "$app_id" From b8798b2a76c00738aeb6534866b04ddd1bbd14bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 11:26:11 +0200 Subject: [PATCH 35/45] fix: cleanup --- .github/workflows/publish_v2_layer.yml | 48 +++++++------------- .github/workflows/reusable_deploy_v2_sar.yml | 8 ---- 2 files changed, 17 insertions(+), 39 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index c68e75af264..7a26ebc7a90 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -10,10 +10,10 @@ on: latest_published_version: description: "Latest PyPi published version to rebuild latest docs for, e.g. v2.0.0" required: true - # workflow_run: - # workflows: ["Publish to PyPi"] - # types: - # - completed + workflow_run: + workflows: ["Publish to PyPi"] + types: + - completed jobs: build-layer: @@ -66,8 +66,7 @@ jobs: cdk --version - name: CDK build run: | - # cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out - cdk synth --context version="git+https://github.com/awslabs/aws-lambda-powertools-python@${{ inputs.latest_published_version }}" -o cdk.out + cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out - name: zip output run: zip -r cdk.out.zip cdk.out - name: Archive CDK artifacts @@ -86,6 +85,16 @@ jobs: artefact-name: "cdk-layer-artefact" environment: "layer-beta" + deploy-prod: + needs: + - deploy-beta + uses: ./.github/workflows/reusable_deploy_layer_stack.yml + secrets: inherit + with: + stage: "PROD" + artefact-name: "cdk-layer-artefact" + environment: "layer-prod" + deploy-sar-beta: needs: - build-layer @@ -95,8 +104,7 @@ jobs: stage: "BETA" artefact-name: "cdk-layer-artefact" environment: "layer-beta" - package-version: 2.0.0 - # package-version: ${{ needs.build-layer.outputs.release-tag-version }} + package-version: ${{ needs.build-layer.outputs.release-tag-version }} deploy-sar-prod: needs: @@ -107,26 +115,4 @@ jobs: stage: "PROD" artefact-name: "cdk-layer-artefact" environment: "layer-prod" - package-version: 2.0.0 - # package-version: ${{ needs.build-layer.outputs.release-tag-version }} - - # deploy-prod: - # needs: - # - deploy-beta - # uses: ./.github/workflows/reusable_deploy_layer_stack.yml - # secrets: inherit - # with: - # stage: "PROD" - # artefact-name: "cdk-layer-artefact" - # environment: "layer-prod" - - # deploy-sar-prod: - # needs: - # - build-layer - # uses: ./.github/workflows/reusable_deploy_v2_sar.yml - # secrets: inherit - # with: - # stage: "PROD" - # artefact-name: "cdk-layer-artefact" - # environment: "layer-prod" - # package-version: ${{ needs.build-layer.outputs.release-tag-version }} + package-version: ${{ needs.build-layer.outputs.release-tag-version }} diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 815420e8cea..8654611a9fe 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -124,11 +124,3 @@ jobs: exit 1 fi echo "Deployment successful" - - name: Cleanup BETA SAR - if: ${{ inputs.stage == 'BETA' }} - run: | - app_id=$(aws serverlessrepo list-applications --output text --query "Applications[?ApplicationId=='arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }}'].ApplicationId") - if [ -z "${app_id}" ]; then - echo "Could not find the SAR that we just deployed." - fi - aws serverlessrepo delete-application --application-id "$app_id" From 136d0e3d62546c6c49ec2d4dbb77538779bec017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 11:26:47 +0200 Subject: [PATCH 36/45] fix: comment out deploy production layer for the time being --- .github/workflows/publish_v2_layer.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 7a26ebc7a90..a7d6efdee04 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -85,15 +85,15 @@ jobs: artefact-name: "cdk-layer-artefact" environment: "layer-beta" - deploy-prod: - needs: - - deploy-beta - uses: ./.github/workflows/reusable_deploy_layer_stack.yml - secrets: inherit - with: - stage: "PROD" - artefact-name: "cdk-layer-artefact" - environment: "layer-prod" + # deploy-prod: + # needs: + # - deploy-beta + # uses: ./.github/workflows/reusable_deploy_layer_stack.yml + # secrets: inherit + # with: + # stage: "PROD" + # artefact-name: "cdk-layer-artefact" + # environment: "layer-prod" deploy-sar-beta: needs: From b98d539a4de941c4f84528b509b4302557d8b516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 11:44:17 +0200 Subject: [PATCH 37/45] fix: cleanup --- .github/workflows/publish_v2_layer.yml | 11 +++++------ .github/workflows/reusable_deploy_v2_sar.yml | 20 +++++++++----------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index a7d6efdee04..e77b754cc56 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -10,10 +10,10 @@ on: latest_published_version: description: "Latest PyPi published version to rebuild latest docs for, e.g. v2.0.0" required: true - workflow_run: - workflows: ["Publish to PyPi"] - types: - - completed + # workflow_run: + # workflows: ["Publish to PyPi"] + # types: + # - completed jobs: build-layer: @@ -65,8 +65,7 @@ jobs: npm install -g aws-cdk@2.44.0 cdk --version - name: CDK build - run: | - cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out + run: cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out - name: zip output run: zip -r cdk.out.zip cdk.out - name: Archive CDK artifacts diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 8654611a9fe..339fe8e181f 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -64,9 +64,8 @@ jobs: with: name: ${{ inputs.artefact-name }} - name: Unzip artefact - run: | - unzip cdk.out.zip - - name: Set SAR name + run: unzip cdk.out.zip + - name: Configure SAR name run: | NAME=${{ env.SAR_NAME }} if [[ "${{ inputs.stage }}" == "BETA" ]]; then @@ -83,13 +82,6 @@ jobs: cp README.md LICENSE "./cdk.out/$asset/" sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" - - name: Publish SAR - if: ${{ inputs.stage == 'PROD' }} - run: | - # wait until sar registers the app, otherwise it fails to make it public - sleep 15 - echo "Make SAR app public" - aws serverlessrepo put-application-policy --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --statements Principals='*',Actions=Deploy - name: Deploy BETA canary if: ${{ inputs.stage == 'BETA' }} run: | @@ -101,7 +93,6 @@ jobs: echo "Check if stack does not exist" stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}" --output text) - echo "$stack_exists" if [[ -n "$stack_exists" ]] ; then echo "Found test deployment stack, removing..." aws cloudformation delete-stack --stack-name "$STACK_NAME" @@ -124,3 +115,10 @@ jobs: exit 1 fi echo "Deployment successful" + - name: Publish SAR + if: ${{ inputs.stage == 'PROD' }} + run: | + # wait until SAR registers the app, otherwise it fails to make it public + sleep 15 + echo "Make SAR app public" + aws serverlessrepo put-application-policy --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --statements Principals='*',Actions=Deploy From 987b524a7f8f5a9489916624970a2d25d3c233ab Mon Sep 17 00:00:00 2001 From: Ruben Fonseca Date: Thu, 13 Oct 2022 14:06:59 +0200 Subject: [PATCH 38/45] chore: apply suggestions from code review Co-authored-by: Heitor Lessa --- .github/workflows/publish_v2_layer.yml | 6 ++---- .github/workflows/reusable_deploy_v2_sar.yml | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index e77b754cc56..1a7a49b06f7 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -95,8 +95,7 @@ jobs: # environment: "layer-prod" deploy-sar-beta: - needs: - - build-layer + needs: build-layer uses: ./.github/workflows/reusable_deploy_v2_sar.yml secrets: inherit with: @@ -106,8 +105,7 @@ jobs: package-version: ${{ needs.build-layer.outputs.release-tag-version }} deploy-sar-prod: - needs: - - deploy-sar-beta + needs: deploy-sar-beta uses: ./.github/workflows/reusable_deploy_v2_sar.yml secrets: inherit with: diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 339fe8e181f..5e474754cab 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -69,7 +69,7 @@ jobs: run: | NAME=${{ env.SAR_NAME }} if [[ "${{ inputs.stage }}" == "BETA" ]]; then - NAME="test-$NAME" + NAME="test-${NAME}" fi echo SAR_NAME="${NAME}" >> "$GITHUB_ENV" - name: Adds arm64 suffix to SAR name @@ -93,6 +93,7 @@ jobs: echo "Check if stack does not exist" stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}" --output text) + if [[ -n "$stack_exists" ]] ; then echo "Found test deployment stack, removing..." aws cloudformation delete-stack --stack-name "$STACK_NAME" From 0f62b6e45fb53280563eebbcf263498f560df818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 14:17:52 +0200 Subject: [PATCH 39/45] chore: add comments to the SAR workflow --- .github/workflows/reusable_deploy_v2_sar.yml | 46 +++++++++++++------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 5e474754cab..ea718a538b6 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -1,14 +1,25 @@ name: Deploy V2 SAR +# SAR deployment process +# +# 1. This workflow starts after the layer artifact is produced on `publish_v2_layer` +# 2. We use the same layer artifact to ensure the SAR app is consistent with the published Lambda Layer +# 3. We publish the SAR for both x86_64 and arm64 (see `matrix` section) +# 4. Publishing the SAR app is done with the help of `sam package` and `sam publish` +# 5. In order to test if the SAR app works, we deploy a Canary stack on the BETA environment +# 5.1. The process starts by removing the previous canary to ensure we can do a clean SAR install +# 5.2. The deployed canary is retained on the account for debugging purposes +# 6. Finally the published SAR app is made public on the PROD environment + permissions: id-token: write contents: read env: - SAM_VERSION: 2.44.0 NODE_VERSION: 16.12 AWS_REGION: eu-west-1 SAR_NAME: aws-lambda-powertools-python-layer-v2 + TEST_STACK_NAME: serverlessrepo-v2-powertools-layer-test-stack on: workflow_call: @@ -67,50 +78,55 @@ jobs: run: unzip cdk.out.zip - name: Configure SAR name run: | - NAME=${{ env.SAR_NAME }} if [[ "${{ inputs.stage }}" == "BETA" ]]; then - NAME="test-${NAME}" + SAR_NAME="test-${SAR_NAME}" fi - echo SAR_NAME="${NAME}" >> "$GITHUB_ENV" + echo SAR_NAME="${SAR_NAME}" >> "$GITHUB_ENV" - name: Adds arm64 suffix to SAR name if: ${{ matrix.architecture == 'arm64' }} run: echo SAR_NAME="${SAR_NAME}-arm64" >> "$GITHUB_ENV" - name: Deploy SAR run: | + # From the generated LayerStack cdk.out artifact, find the layer asset path for the correct architecture. + # We'll use this as the source directory of our SAR. This way we are re-using the same layer asset for our SAR. asset=$(jq -jc '.Resources[] | select(.Properties.CompatibleArchitectures == ["${{ matrix.architecture }}"]) | .Metadata."aws:asset:path"' cdk.out/LayerV2Stack.template.json) + + # fill in the SAR SAM template sed -e "s||${{ inputs.package-version }}|g" -e "s//${{ env.SAR_NAME }}/g" -e "s||./cdk.out/$asset|g" layer/sar/template.txt > template.yml + + # SAR needs a README and a LICENSE, so just copy the ones from the repo cp README.md LICENSE "./cdk.out/$asset/" + + # Package the SAR to our SAR S3 bucket, and publish it sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket ${{ secrets.AWS_SAR_S3_BUCKET }} sam publish --template packaged.yml --region "$AWS_REGION" - name: Deploy BETA canary if: ${{ inputs.stage == 'BETA' }} run: | - STACK_NAME=serverlessrepo-v2-powertools-layer-test-stack - if [[ "${{ matrix.architecture }}" == "arm64" ]]; then - STACK_NAME="${STACK_NAME}-arm64" + TEST_STACK_NAME="${TEST_STACK_NAME}-arm64" fi echo "Check if stack does not exist" - stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}" --output text) + stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$TEST_STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}" --output text) if [[ -n "$stack_exists" ]] ; then echo "Found test deployment stack, removing..." - aws cloudformation delete-stack --stack-name "$STACK_NAME" - aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" + aws cloudformation delete-stack --stack-name "$TEST_STACK_NAME" + aws cloudformation wait stack-delete-complete --stack-name "$TEST_STACK_NAME" fi echo "Creating canary stack" - echo "Stack name: $STACK_NAME" - aws serverlessrepo create-cloud-formation-change-set --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --stack-name "${STACK_NAME/serverlessrepo-/}" --capabilities CAPABILITY_NAMED_IAM - CHANGE_SET_ID=$(aws cloudformation list-change-sets --stack-name "$STACK_NAME" --query 'Summaries[*].ChangeSetId' --output text) + echo "Stack name: $TEST_STACK_NAME" + aws serverlessrepo create-cloud-formation-change-set --application-id arn:aws:serverlessrepo:${{ env.AWS_REGION }}:${{ steps.aws-credentials-sar-role.outputs.aws-account-id }}:applications/${{ env.SAR_NAME }} --stack-name "${TEST_STACK_NAME/serverlessrepo-/}" --capabilities CAPABILITY_NAMED_IAM + CHANGE_SET_ID=$(aws cloudformation list-change-sets --stack-name "$TEST_STACK_NAME" --query 'Summaries[*].ChangeSetId' --output text) aws cloudformation wait change-set-create-complete --change-set-name "$CHANGE_SET_ID" aws cloudformation execute-change-set --change-set-name "$CHANGE_SET_ID" - aws cloudformation wait stack-create-complete --stack-name "$STACK_NAME" + aws cloudformation wait stack-create-complete --stack-name "$TEST_STACK_NAME" echo "Waiting until stack deployment completes..." echo "Exit with error if stack is not in CREATE_COMPLETE" - stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}") + stack_exists=$(aws cloudformation list-stacks --query "StackSummaries[?(StackName == '$TEST_STACK_NAME' && StackStatus == 'CREATE_COMPLETE')].{StackId:StackId, StackName:StackName, CreationTime:CreationTime, StackStatus:StackStatus}") if [[ -z "$stack_exists" ]] ; then echo "Could find successful deployment, exit error..." exit 1 From f4cc4124a893b5510d1217c1cbad213f543c74c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 14:25:49 +0200 Subject: [PATCH 40/45] fix: stop using deprecated set-output --- .github/workflows/publish_v2_layer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 1a7a49b06f7..98a8d18a456 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -52,7 +52,7 @@ jobs: RELEASE_INPUT=${{ inputs.latest_published_version }} LATEST_TAG=$(git describe --tag --abbrev=0) RELEASE_TAG_VERSION=${RELEASE_INPUT:-$LATEST_TAG} - echo "::set-output name=release-tag-version::$RELEASE_TAG_VERSION" + echo release-tag-version="$RELEASE_TAG_VERSION" >> "$GITHUB_OUTPUT" echo RELEASE_TAG_VERSION="${RELEASE_TAG_VERSION:1}" >> "$GITHUB_ENV" - name: Set up QEMU uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v2.0.0 From 9f10f5e552669a2a72266a91a1568be110d0f65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BAben=20Fonseca?= Date: Thu, 13 Oct 2022 14:29:49 +0200 Subject: [PATCH 41/45] fix: remove unecessary ENV --- .github/workflows/publish_v2_layer.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish_v2_layer.yml b/.github/workflows/publish_v2_layer.yml index 98a8d18a456..de04566c79e 100644 --- a/.github/workflows/publish_v2_layer.yml +++ b/.github/workflows/publish_v2_layer.yml @@ -23,7 +23,7 @@ jobs: run: working-directory: ./layer outputs: - release-tag-version: ${{ steps.release-notes-tag.outputs.release-tag-version }} + release-tag-version: ${{ steps.release-notes-tag.outputs.RELEASE_TAG_VERSION }} steps: - name: checkout uses: actions/checkout@v3 @@ -52,8 +52,7 @@ jobs: RELEASE_INPUT=${{ inputs.latest_published_version }} LATEST_TAG=$(git describe --tag --abbrev=0) RELEASE_TAG_VERSION=${RELEASE_INPUT:-$LATEST_TAG} - echo release-tag-version="$RELEASE_TAG_VERSION" >> "$GITHUB_OUTPUT" - echo RELEASE_TAG_VERSION="${RELEASE_TAG_VERSION:1}" >> "$GITHUB_ENV" + echo RELEASE_TAG_VERSION="${RELEASE_TAG_VERSION:1}" >> "$GITHUB_OUTPUT" - name: Set up QEMU uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v2.0.0 # NOTE: we need QEMU to build Layer against a different architecture (e.g., ARM) @@ -65,7 +64,7 @@ jobs: npm install -g aws-cdk@2.44.0 cdk --version - name: CDK build - run: cdk synth --context version="$RELEASE_TAG_VERSION" -o cdk.out + run: cdk synth --context version="${{ steps.release-notes-tag.outputs.RELEASE_TAG_VERSION }}" -o cdk.out - name: zip output run: zip -r cdk.out.zip cdk.out - name: Archive CDK artifacts From 9abec5a17775ceb6fcb5c709cd94dc421db55494 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Thu, 13 Oct 2022 15:23:37 +0200 Subject: [PATCH 42/45] chore: line editing --- .github/workflows/reusable_deploy_v2_sar.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index ea718a538b6..e7942e0663c 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -6,9 +6,9 @@ name: Deploy V2 SAR # 2. We use the same layer artifact to ensure the SAR app is consistent with the published Lambda Layer # 3. We publish the SAR for both x86_64 and arm64 (see `matrix` section) # 4. Publishing the SAR app is done with the help of `sam package` and `sam publish` -# 5. In order to test if the SAR app works, we deploy a Canary stack on the BETA environment +# 5. We deploy a Canary stack in the BETA environment to test the SAR App # 5.1. The process starts by removing the previous canary to ensure we can do a clean SAR install -# 5.2. The deployed canary is retained on the account for debugging purposes +# 5.2. The canary is in the account for debugging purposes # 6. Finally the published SAR app is made public on the PROD environment permissions: From 578448a17d7891607895d3a74e46f56e2ac52c41 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Thu, 13 Oct 2022 15:24:30 +0200 Subject: [PATCH 43/45] chore: line editing use active voice to shorten the length --- .github/workflows/reusable_deploy_v2_sar.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index e7942e0663c..297b8d9531a 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -5,7 +5,7 @@ name: Deploy V2 SAR # 1. This workflow starts after the layer artifact is produced on `publish_v2_layer` # 2. We use the same layer artifact to ensure the SAR app is consistent with the published Lambda Layer # 3. We publish the SAR for both x86_64 and arm64 (see `matrix` section) -# 4. Publishing the SAR app is done with the help of `sam package` and `sam publish` +# 4. We use `sam package` and `sam publish` to publish the SAR app # 5. We deploy a Canary stack in the BETA environment to test the SAR App # 5.1. The process starts by removing the previous canary to ensure we can do a clean SAR install # 5.2. The canary is in the account for debugging purposes From 57b40abc151d829282a757d3e06cea85cb81b7e4 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Thu, 13 Oct 2022 15:28:32 +0200 Subject: [PATCH 44/45] chore: line editing part2 remove fragmented sentence by combining them. --- .github/workflows/reusable_deploy_v2_sar.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 297b8d9531a..4810527c4de 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -6,9 +6,7 @@ name: Deploy V2 SAR # 2. We use the same layer artifact to ensure the SAR app is consistent with the published Lambda Layer # 3. We publish the SAR for both x86_64 and arm64 (see `matrix` section) # 4. We use `sam package` and `sam publish` to publish the SAR app -# 5. We deploy a Canary stack in the BETA environment to test the SAR App -# 5.1. The process starts by removing the previous canary to ensure we can do a clean SAR install -# 5.2. The canary is in the account for debugging purposes +# 5. We remove the previous Canary stack (if present) and deploy a new one to test the SAR App. We retain the for debugging purposes # 6. Finally the published SAR app is made public on the PROD environment permissions: From 54f79f8a9c7943f8d16bed5e29d2ac0df5c6b213 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Thu, 13 Oct 2022 15:29:04 +0200 Subject: [PATCH 45/45] chore: typo from line-editing --- .github/workflows/reusable_deploy_v2_sar.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_deploy_v2_sar.yml b/.github/workflows/reusable_deploy_v2_sar.yml index 4810527c4de..acb28179efe 100644 --- a/.github/workflows/reusable_deploy_v2_sar.yml +++ b/.github/workflows/reusable_deploy_v2_sar.yml @@ -6,7 +6,7 @@ name: Deploy V2 SAR # 2. We use the same layer artifact to ensure the SAR app is consistent with the published Lambda Layer # 3. We publish the SAR for both x86_64 and arm64 (see `matrix` section) # 4. We use `sam package` and `sam publish` to publish the SAR app -# 5. We remove the previous Canary stack (if present) and deploy a new one to test the SAR App. We retain the for debugging purposes +# 5. We remove the previous Canary stack (if present) and deploy a new one to test the SAR App. We retain the Canary in the account for debugging purposes # 6. Finally the published SAR app is made public on the PROD environment permissions: