From 36bdd33d7931d0c00617a81a9b207c8c18f21f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Tue, 8 Oct 2024 13:57:04 +0200 Subject: [PATCH 01/25] add docker fine grained permission to the plugin --- Examples/APIGateway/README.md | 2 +- Examples/HelloWorld/README.md | 2 +- Package.swift | 8 +++++++- Plugins/AWSLambdaPackager/Plugin.swift | 2 +- .../Resources/code/04-01-02-plugin-archive.sh | 2 +- .../Resources/code/04-01-03-plugin-archive.sh | 2 +- .../Resources/code/04-01-04-plugin-archive.sh | 2 +- .../Documentation.docc/quick-setup.md | 2 +- readme.md | 2 +- 9 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Examples/APIGateway/README.md b/Examples/APIGateway/README.md index ca731ec6..f534f9ba 100644 --- a/Examples/APIGateway/README.md +++ b/Examples/APIGateway/README.md @@ -22,7 +22,7 @@ To build the package, type the following commands. ```bash swift build -swift package archive --disable-sandbox +swift package archive --allow-network-access docker ``` If there is no error, there is a ZIP file ready to deploy. diff --git a/Examples/HelloWorld/README.md b/Examples/HelloWorld/README.md index e3e2e08b..7709c3ef 100644 --- a/Examples/HelloWorld/README.md +++ b/Examples/HelloWorld/README.md @@ -18,7 +18,7 @@ To build & archive the package, type the following commands. ```bash swift build -swift package archive --disable-sandbox +swift package archive --allow-network-access docker ``` If there is no error, there is a ZIP file ready to deploy. diff --git a/Package.swift b/Package.swift index fe9533bd..129ac3bb 100644 --- a/Package.swift +++ b/Package.swift @@ -55,7 +55,13 @@ let package = Package( verb: "archive", description: "Archive the Lambda binary and prepare it for uploading to AWS. Requires docker on macOS or non Amazonlinux 2 distributions." - ) + ), +permissions: [ + .allowNetworkConnections( + scope: .docker, + reason: "This plugin uses Docker to create the AWS Lambda ZIP package." + ) + ] ) ), .testTarget( diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 01c28d47..916e7bc7 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -287,7 +287,7 @@ struct AWSLambdaPackager: CommandPlugin { REQUIREMENTS: To use this plugin, you must have docker installed and started. - USAGE: swift package --disable-sandbox archive [--help] [--verbose] + USAGE: swift package --allow-network-access docker archive [--help] [--verbose] [--output-directory ] [--products ] [--configuration debug | release] diff --git a/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-02-plugin-archive.sh b/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-02-plugin-archive.sh index 41e7a628..e1b4aa05 100644 --- a/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-02-plugin-archive.sh +++ b/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-02-plugin-archive.sh @@ -1,2 +1,2 @@ -swift package --disable-sandbox plugin archive +swift package --allow-network-access docker plugin archive diff --git a/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-03-plugin-archive.sh b/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-03-plugin-archive.sh index 9878f478..37a5cd1b 100644 --- a/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-03-plugin-archive.sh +++ b/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-03-plugin-archive.sh @@ -1,4 +1,4 @@ -swift package --disable-sandbox plugin archive +swift package --allow-network-access docker archive ------------------------------------------------------------------------- building "squarenumberlambda" in docker diff --git a/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-04-plugin-archive.sh b/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-04-plugin-archive.sh index 7652bf1c..b35c9f65 100644 --- a/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-04-plugin-archive.sh +++ b/Sources/AWSLambdaRuntimeCore/Documentation.docc/Resources/code/04-01-04-plugin-archive.sh @@ -1,4 +1,4 @@ -swift package --disable-sandbox plugin archive +swift package --allow-network-access docker archive ------------------------------------------------------------------------- building "squarenumberlambda" in docker diff --git a/Sources/AWSLambdaRuntimeCore/Documentation.docc/quick-setup.md b/Sources/AWSLambdaRuntimeCore/Documentation.docc/quick-setup.md index 151fb8c3..60065baa 100644 --- a/Sources/AWSLambdaRuntimeCore/Documentation.docc/quick-setup.md +++ b/Sources/AWSLambdaRuntimeCore/Documentation.docc/quick-setup.md @@ -106,7 +106,7 @@ AWS Lambda runtime runs on Amazon Linux. You must compile your code for Amazon L > Be sure to have [Docker](https://docs.docker.com/desktop/install/mac-install/) installed for this step. ```sh -swift package --disable-sandbox plugin archive +swift package --allow-network-access docker archive ------------------------------------------------------------------------- building "squarenumberlambda" in docker diff --git a/readme.md b/readme.md index fd68681c..e33b5379 100644 --- a/readme.md +++ b/readme.md @@ -83,7 +83,7 @@ try await runtime.run() ```bash swift build -swift package archive --disable-sandbox +swift package archive --allow-network-access docker ``` If there is no error, there is a ZIP file ready to deploy. From 7b28a796f779978e10fd4e5e9e1d86748ae76d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Tue, 8 Oct 2024 14:02:29 +0200 Subject: [PATCH 02/25] swift-format --- Package.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 129ac3bb..1c97e40d 100644 --- a/Package.swift +++ b/Package.swift @@ -56,12 +56,12 @@ let package = Package( description: "Archive the Lambda binary and prepare it for uploading to AWS. Requires docker on macOS or non Amazonlinux 2 distributions." ), -permissions: [ + permissions: [ .allowNetworkConnections( scope: .docker, reason: "This plugin uses Docker to create the AWS Lambda ZIP package." ) - ] + ] ) ), .testTarget( From 32b99bd2358caaa2c46d44f5b992204685435780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Tue, 8 Oct 2024 18:25:57 +0200 Subject: [PATCH 03/25] initial commit to fix docker compose --- docker/docker-compose.al2.510.yaml | 18 ------------------ docker/docker-compose.al2.57.yaml | 18 ------------------ docker/docker-compose.al2.58.yaml | 18 ------------------ docker/docker-compose.al2.59.yaml | 18 ------------------ docker/docker-compose.al2.6.0.yaml | 18 ++++++++++++++++++ docker/docker-compose.yaml | 18 +++++++++--------- 6 files changed, 27 insertions(+), 81 deletions(-) delete mode 100644 docker/docker-compose.al2.510.yaml delete mode 100644 docker/docker-compose.al2.57.yaml delete mode 100644 docker/docker-compose.al2.58.yaml delete mode 100644 docker/docker-compose.al2.59.yaml create mode 100644 docker/docker-compose.al2.6.0.yaml diff --git a/docker/docker-compose.al2.510.yaml b/docker/docker-compose.al2.510.yaml deleted file mode 100644 index a897f987..00000000 --- a/docker/docker-compose.al2.510.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: swift-aws-lambda:al2-5.10 - build: - args: - base_image: "swiftlang/swift:nightly-5.10-amazonlinux2" - - test: - image: swift-aws-lambda:al2-5.10 - - test-examples: - image: swift-aws-lambda:al2-5.10 - - shell: - image: swift-aws-lambda:al2-5.10 diff --git a/docker/docker-compose.al2.57.yaml b/docker/docker-compose.al2.57.yaml deleted file mode 100644 index 1b19f1f0..00000000 --- a/docker/docker-compose.al2.57.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: swift-aws-lambda:al2-5.7 - build: - args: - swift_version: "5.7" - - test: - image: swift-aws-lambda:al2-5.7 - - test-examples: - image: swift-aws-lambda:al2-5.7 - - shell: - image: swift-aws-lambda:al2-5.7 diff --git a/docker/docker-compose.al2.58.yaml b/docker/docker-compose.al2.58.yaml deleted file mode 100644 index 6127c65c..00000000 --- a/docker/docker-compose.al2.58.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: swift-aws-lambda:al2-5.8 - build: - args: - swift_version: "5.8" - - test: - image: swift-aws-lambda:al2-5.8 - - test-examples: - image: swift-aws-lambda:al2-5.8 - - shell: - image: swift-aws-lambda:al2-5.8 diff --git a/docker/docker-compose.al2.59.yaml b/docker/docker-compose.al2.59.yaml deleted file mode 100644 index edea9327..00000000 --- a/docker/docker-compose.al2.59.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: swift-aws-lambda:al2-5.9 - build: - args: - swift_version: "5.9" - - test: - image: swift-aws-lambda:al2-5.9 - - test-examples: - image: swift-aws-lambda:al2-5.9 - - shell: - image: swift-aws-lambda:al2-5.9 diff --git a/docker/docker-compose.al2.6.0.yaml b/docker/docker-compose.al2.6.0.yaml new file mode 100644 index 00000000..25366209 --- /dev/null +++ b/docker/docker-compose.al2.6.0.yaml @@ -0,0 +1,18 @@ +version: "3" + +services: + + runtime-setup: + image: swift-aws-lambda:al2-6.0 + build: + args: + base_image: "swiftlang/swift:nightly-6.0-amazonlinux2" + + test: + image: swift-aws-lambda:al2-6.0 + + test-examples: + image: swift-aws-lambda:al2-6.0 + + shell: + image: swift-aws-lambda:al2-6.0 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 32507dcf..94478301 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -24,7 +24,11 @@ services: soundness: <<: *common - command: /bin/bash -cl "./scripts/soundness.sh" + command: >- + /bin/bash -clx " + curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-swift-format.sh | bash && + curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-license-header.sh | bash + " test: <<: *common @@ -34,14 +38,10 @@ services: <<: *common command: >- /bin/bash -clx " - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Benchmark && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Deployment && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Echo && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/ErrorHandling && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Foundation && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/JSON && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/LocalDebugging/MyLambda && - LAMBDA_USE_LOCAL_DEPS=true swift test --package-path Examples/Testing + LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/APIGateway && + LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/AWSSDK && + LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/HelloWorld && + LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Soto && " # util From 83ed7057f4403b3ceb828ff4197a99fd2e79b232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Tue, 8 Oct 2024 19:25:39 +0200 Subject: [PATCH 04/25] update docker files --- docker/Dockerfile | 9 +-------- docker/docker-compose.yaml | 5 ++--- readme.md | 8 ++++++++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ec97cef2..96e416c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG swift_version=5.7 +ARG swift_version=6.0 ARG base_image=swift:$swift_version-amazonlinux2 FROM $base_image # needed to do again after FROM due to docker limitation @@ -11,10 +11,3 @@ RUN yum install -y lsof dnsutils netcat-openbsd net-tools curl jq # used by inte # tools RUN mkdir -p $HOME/.tools RUN echo 'export PATH="$HOME/.tools:$PATH"' >> $HOME/.profile - -# swiftformat (until part of the toolchain) - -ARG swiftformat_version=0.50.1 -RUN git clone --branch $swiftformat_version --depth 1 https://github.com/nicklockwood/SwiftFormat $HOME/.tools/swift-format -RUN cd $HOME/.tools/swift-format && swift build -c release -RUN ln -s $HOME/.tools/swift-format/.build/release/swiftformat $HOME/.tools/swiftformat diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 94478301..b39e350e 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,6 @@ # this file is not designed to be run directly # instead, use the docker-compose.. files -# eg docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.al2.57.yaml run test -version: "3" +# eg docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.al2.6.0.yaml run test services: @@ -41,7 +40,7 @@ services: LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/APIGateway && LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/AWSSDK && LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/HelloWorld && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Soto && + LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Soto " # util diff --git a/readme.md b/readme.md index e33b5379..1de0000e 100644 --- a/readme.md +++ b/readme.md @@ -163,3 +163,11 @@ tbd + link to docc ### Background Tasks tbd + link to docc + +### Development + +Your contributions are welcome. If you open a pull request, be sure to ensure it passes the soundness checks + +``` +docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.al2.6.0.yaml run soundness +``` From 7d26fe7337e65cba64c9f5f0913c0aeb5f262a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Tue, 8 Oct 2024 21:11:02 +0200 Subject: [PATCH 05/25] add GH action to compile examples on PR --- .github/workflows/pull_request.yml | 8 ++++++++ scripts/integration_tests.sh | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 scripts/integration_tests.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 39e033e2..8c8b50ae 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -24,6 +24,14 @@ jobs: linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error" linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error" + integration-tests: + name: Integration Tests + # Workaround https://github.com/nektos/act/issues/1875 + uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main + with: + name: "Integration tests" + matrix_linux_command: "./scripts/integration_tests.sh" + swift-6-language-mode: name: Swift 6 Language Mode uses: apple/swift-nio/.github/workflows/swift_6_language_mode.yml@main diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh new file mode 100644 index 00000000..f87c9527 --- /dev/null +++ b/scripts/integration_tests.sh @@ -0,0 +1,21 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftAWSLambdaRuntime open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftAWSLambdaRuntime project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set +ex + +LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/APIGateway +LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/AWSSDK +LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/HelloWorld +LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Soto From 686337aea02ef12eef16ae5728e33a79119bc8ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Tue, 8 Oct 2024 21:13:56 +0200 Subject: [PATCH 06/25] make script executable --- scripts/integration_tests.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/integration_tests.sh diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh old mode 100644 new mode 100755 From 34ffd094ddea581b485b3e95e45813eb26449a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Tue, 8 Oct 2024 21:14:35 +0200 Subject: [PATCH 07/25] disable integration tests on Swift < 6.0 --- .github/workflows/pull_request.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5002c532..9bc435c9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -29,8 +29,11 @@ jobs: # Workaround https://github.com/nektos/act/issues/1875 uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main with: - name: "Integration tests" - matrix_linux_command: "./scripts/integration_tests.sh" + name: "Integration tests" + matrix_linux_command: "./scripts/integration_tests.sh" + linux_5_8_enabled: false + linux_5_9_enabled: false + linux_5_10_enabled: false swift-6-language-mode: name: Swift 6 Language Mode From cffbfadf49ce886fc54451076d1776bba218007e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 10:48:24 +0200 Subject: [PATCH 08/25] remove old docker infrastructure --- docker/Dockerfile | 13 -------- docker/docker-compose.al2.6.0.yaml | 18 ----------- docker/docker-compose.al2.main.yaml | 18 ----------- docker/docker-compose.yaml | 50 ----------------------------- 4 files changed, 99 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100644 docker/docker-compose.al2.6.0.yaml delete mode 100644 docker/docker-compose.al2.main.yaml delete mode 100644 docker/docker-compose.yaml diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 96e416c4..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG swift_version=6.0 -ARG base_image=swift:$swift_version-amazonlinux2 -FROM $base_image -# needed to do again after FROM due to docker limitation -ARG swift_version - -# dependencies -RUN yum install -y wget perl-Digest-SHA -RUN yum install -y lsof dnsutils netcat-openbsd net-tools curl jq # used by integration tests - -# tools -RUN mkdir -p $HOME/.tools -RUN echo 'export PATH="$HOME/.tools:$PATH"' >> $HOME/.profile diff --git a/docker/docker-compose.al2.6.0.yaml b/docker/docker-compose.al2.6.0.yaml deleted file mode 100644 index 25366209..00000000 --- a/docker/docker-compose.al2.6.0.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: swift-aws-lambda:al2-6.0 - build: - args: - base_image: "swiftlang/swift:nightly-6.0-amazonlinux2" - - test: - image: swift-aws-lambda:al2-6.0 - - test-examples: - image: swift-aws-lambda:al2-6.0 - - shell: - image: swift-aws-lambda:al2-6.0 diff --git a/docker/docker-compose.al2.main.yaml b/docker/docker-compose.al2.main.yaml deleted file mode 100644 index b2f890c1..00000000 --- a/docker/docker-compose.al2.main.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: swift-aws-lambda:al2-main - build: - args: - base_image: "swiftlang/swift:nightly-main-amazonlinux2" - - test: - image: swift-aws-lambda:al2-main - - test-examples: - image: swift-aws-lambda:al2-main - - shell: - image: swift-aws-lambda:al2-main diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml deleted file mode 100644 index b39e350e..00000000 --- a/docker/docker-compose.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# this file is not designed to be run directly -# instead, use the docker-compose.. files -# eg docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.al2.6.0.yaml run test - -services: - - runtime-setup: - image: swift-aws-lambda:default - build: - context: . - dockerfile: Dockerfile - - common: &common - image: swift-aws-lambda:default - depends_on: [runtime-setup] - volumes: - - ~/.ssh:/root/.ssh - - ..:/code:z - working_dir: /code - cap_drop: - - CAP_NET_RAW - - CAP_NET_BIND_SERVICE - - soundness: - <<: *common - command: >- - /bin/bash -clx " - curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-swift-format.sh | bash && - curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-license-header.sh | bash - " - - test: - <<: *common - command: /bin/bash -cl "swift test -Xswiftc -warnings-as-errors $${SANITIZER_ARG-}" - - test-examples: - <<: *common - command: >- - /bin/bash -clx " - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/APIGateway && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/AWSSDK && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/HelloWorld && - LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Soto - " - - # util - - shell: - <<: *common - entrypoint: /bin/bash -l From ed39d47744551427b9fb939e5572cdabeac7faff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 10:48:47 +0200 Subject: [PATCH 09/25] automate the discovery of examples --- scripts/integration_tests.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh index f87c9527..816d666e 100755 --- a/scripts/integration_tests.sh +++ b/scripts/integration_tests.sh @@ -13,9 +13,12 @@ ## ##===----------------------------------------------------------------------===## -set +ex +set +x -e -LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/APIGateway -LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/AWSSDK -LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/HelloWorld -LAMBDA_USE_LOCAL_DEPS=true swift build --package-path Examples/Soto +for EXAMPLE in $(find Examples -type d -d 1); +do + echo "Building $EXAMPLE" + pushd $EXAMPLE + LAMBDA_USE_LOCAL_DEPS=../.. swift build + popd +done From f37846d0be26287f19c7e236ab184b460af64c59 Mon Sep 17 00:00:00 2001 From: aryan-25 Date: Wed, 9 Oct 2024 09:05:02 +0100 Subject: [PATCH 10/25] v2 API Proposal Document (#339) Co-authored-by: Fabian Fett --- .../Proposals/0001-v2-api.md | 905 ++++++++++++++++++ 1 file changed, 905 insertions(+) create mode 100644 Sources/AWSLambdaRuntimeCore/Documentation.docc/Proposals/0001-v2-api.md diff --git a/Sources/AWSLambdaRuntimeCore/Documentation.docc/Proposals/0001-v2-api.md b/Sources/AWSLambdaRuntimeCore/Documentation.docc/Proposals/0001-v2-api.md new file mode 100644 index 00000000..e4ff259b --- /dev/null +++ b/Sources/AWSLambdaRuntimeCore/Documentation.docc/Proposals/0001-v2-api.md @@ -0,0 +1,905 @@ +# v2 API proposal for swift-aws-lambda-runtime + +`swift-aws-lambda-runtime` is an important library for the Swift on Server ecosystem. The initial API was written before +async/await was introduced to Swift. When async/await was introduced, shims were added to bridge between the underlying +SwiftNIO `EventLoop` interfaces and async/await. However, just like `gRPC-swift` and `postgres-nio`, we now want to +shift to solely using async/await instead of `EventLoop` interfaces. For this, large parts of the current API have to be +reconsidered. + +## Overview + +Versions: + +- v1 (2024-08-07): Initial version +- v1.1: + - Remove the `reportError(_:)` method from `LambdaResponseStreamWriter` and instead make the `handle(...)` method of + `StreamingLambdaHandler` throwing. + - Remove the `addBackgroundTask(_:)` method from `LambdaContext` due to structured concurrency concerns and introduce + the `LambdaWithBackgroundProcessingHandler` protocol as a solution. + - Introduce `LambdaHandlerAdapter`, which adapts handlers conforming to `LambdaHandler` with + `LambdaWithBackgroundProcessingHandler`. + - Update `LambdaCodableAdapter` to now be generic over any handler conforming to + `LambdaWithBackgroundProcessingHandler` instead of `LambdaHandler`. +- v1.2: + - Remove `~Copyable` from `LambdaResponseStreamWriter` and `LambdaResponseWriter`. Instead throw an error when + `finish()` is called multiple times or when `write`/`writeAndFinish` is called after `finish()`. + +## Motivation + +### Current Limitations + +#### EventLoop interfaces + +The current API extensively uses the `EventLoop` family of interfaces from SwiftNIO in many areas. To use these +interfaces correctly though, it requires developers to exercise great care and understand the various transform methods +that are used to work with `EventLoop`s and `EventLoopFuture`s. This results in a lot of cognitive complexity and makes +the code in the current API hard to reason about and maintain. For these reasons, the overarching trend in the Swift on +Server ecosystem is to shift to newer, more readable, Swift concurrency constructs and de-couple from SwiftNIO's +`EventLoop` interfaces. + +#### No ownership of the main() function + +A Lambda function can currently be implemented through conformance to the various handler protocols defined in +`AWSLambdaRuntimeCore/LambdaHandler`. Each of these protocols have an extension which implements a `static func main()`. +This allows users to annotate their `LambdaHandler` conforming object with `@main`. The `static func main()` calls the +internal `Lambda.run()` function, which starts the Lambda function. Since the `Lambda.run()` method is internal, users +cannot override the default implementation. This has proven challenging for users who want to +[set up global properties before the Lambda starts-up](https://github.com/swift-server/swift-aws-lambda-runtime/issues/265). +Setting up global properties is required to customize the Swift Logging, Metric and Tracing backend. + +#### Non-trivial transition from SimpleLambdaHandler to LambdaHandler + +The `SimpleLambdaHandler` protocol provides a quick and easy way to implement a basic Lambda function. It only requires +an implementation of the `handle` function where the business logic of the Lambda function can be written. +`SimpleLambdaHandler` is perfectly sufficient for small use-cases as the user does not need to spend much time looking +into the library. + +However, `SimpleLambdaHandler` cannot be used when services such as a database client need to be initialized before the +Lambda runtime starts and then also gracefully shutdown prior to the runtime terminating. This is because the only way +to register termination logic is through the `LambdaInitializationContext` (containing a field +`terminator: LambdaTerminator`) which is created and used _internally_ within `LambdaRuntime` and never exposed through +`SimpleLambdaHandler`. For such use-cases, other handler protocols like `LambdaHandler` must be used. `LambdaHandler` +exposes a `context` argument of type `LambdaInitializationContext` through its initializer. Within the initializer, +required services can be initialized and their graceful shutdown logic can be registered with the +`context.terminator.register` function. + +Yet, `LambdaHandler` is quite cumbersome to use in such use-cases as users have to deviate from the established norms of +the Swift on Server ecosystem in order to cleanly manage the lifecycle of the services intended to be used. This is +because the convenient `swift-service-lifecycle` v2 library — which is commonly used for cleanly managing the lifecycles +of required services and widely supported by many libraries — cannot be used in a structured concurrency manner. + +#### Does not integrate well with swift-service-lifecycle in a structured concurrency manner + +The Lambda runtime can only be started using the **internal** `Lambda.run()` function. This function is called by the +`main()` function defined by the `LambdaHandler` protocol, preventing users from injecting initialized services into the +runtime _prior_ to it starting. As shown below, this forces users to use an **unstructured concurrency** approach and +manually initialize services, leading to the issue of the user then perhaps forgetting to gracefully shutdown the +initialized services: + +```swift +struct MyLambda: LambdaHandler { + let pgClient: PostgresClient + + init(context: AWSLambdaRuntimeCore.LambdaInitializationContext) async throws { + /// Instantiate service + let client = PostgresClient(configuration: ...) + + /// Unstructured concurrency to initialize the service + let pgTask = Task { + await client.run() + } + + /// Store the client in `self` so that it can be used in `handle(...)` + self.pgClient = client + + /// !!! Must remember to explicitly register termination logic for PostgresClient !!! + context.terminator.register( + name: "PostgreSQL Client", + handler: { eventLoop in + pgTask.cancel() + return eventLoop.makeFutureWithTask { + await pgTask.value + } + } + ) + } + + func handle(_ event: Event, context: LambdaContext) async throws -> Output { + /// Use the initialized service stored in `self.pgClient` + try await self.pgClient.query(...) + } +} +``` + +#### Verbose Codable support + +In the current API, there are extensions and Codable wrapper classes for decoding events and encoding computed responses +for _each_ different handler protocol and for both `String` and `JSON` formats. This has resulted in a lot of +boilerplate code which can very easily be made generic and simplified in v2. + +### New features + +#### Support response streaming + +In April 2023 +[AWS introduced support for response streaming](https://aws.amazon.com/blogs/compute/introducing-aws-lambda-response-streaming/) +in Lambda. The current API does not support streaming. For v2 we want to change this. + +#### Scheduling background work + +In May +[AWS described in a blog post that you can run background tasks in Lambda](https://aws.amazon.com/blogs/compute/running-code-after-returning-a-response-from-an-aws-lambda-function/) +until the runtime asks for more work from the control plane. We want to support this by adding new API that allows +background processing, even after the response has been returned. + +## Proposed Solution + +### async/await-first API + +Large parts of `Lambda`, `LambdaHandler`, and `LambdaRuntime` will be re-written to use async/await constructs in place +of the `EventLoop` family of interfaces. + +### Providing ownership of main() and support for swift-service-lifecycle + +- Instead of conforming to a handler protocol, users can now create a `LambdaRuntime` by passing in a handler closure. +- `LambdaRuntime` conforms to `ServiceLifecycle.Service` by implementing a `run()` method that contains initialization + and graceful shutdown logic. +- This allows the lifecycle of the `LambdaRuntime` to be managed with `swift-service-lifecycle` _alongside_ and in the + same way the lifecycles of the required services are managed, e.g. + `try await ServiceGroup(services: [postgresClient, ..., lambdaRuntime], ...).run()`. +- Dependencies can now be injected into `LambdaRuntime`. With `swift-service-lifecycle`, services will be initialized + together with `LambdaRuntime`. +- The required services can then be used within the handler in a structured concurrency manner. + `swift-service-lifecycle` takes care of listening for termination signals and terminating the services as well as the + `LambdaRuntime` in correct order. +- `LambdaTerminator` can now be eliminated because its role is replaced with `swift-service-lifecycle`. The termination + logic of the Lambda function will be implemented in the conforming `run()` function of `LambdaRuntime`. + +With this, the earlier code snippet can be replaced with something much easier to read, maintain, and debug: + +```swift +/// Instantiate services +let postgresClient = PostgresClient() + +/// Instantiate LambdaRuntime with a closure handler implementing the business logic of the Lambda function +let runtime = LambdaRuntime { (event: Input, context: LambdaContext) in + /// Use initialized service within the handler + try await postgresClient.query(...) +} + +/// Use ServiceLifecycle to manage the initialization and termination +/// of the services as well as the LambdaRuntime +let serviceGroup = ServiceGroup( + services: [postgresClient, runtime], + configuration: .init(gracefulShutdownSignals: [.sigterm]), + logger: logger +) +try await serviceGroup.run() +``` + +### Simplifying Codable support + +A detailed explanation is provided in the **Codable Support** section. In short, much of the boilerplate code defined +for each handler protocol in `Lambda+Codable` and `Lambda+String` will be replaced with a single `LambdaCodableAdapter` +struct. + +This adapter struct is generic over (1) any handler conforming to a new handler protocol +`LambdaWithBackgroundProcessingHandler`, (2) the user-specified input and output types, and (3) any decoder and encoder +conforming to protocols `LambdaEventDecoder` and `LambdaOutputDecoder`. The adapter will wrap the underlying handler +with encoding/decoding logic. + +## Detailed Solution + +Below are explanations for all types that we want to use in AWS Lambda Runtime v2. + +### LambdaResponseStreamWriter + +We will introduce a new `LambdaResponseStreamWriter` protocol. It is used in the new `StreamingLambdaHandler` (defined +below), which is the new base protocol for the `LambdaRuntime` (defined below as well). + +```swift +/// A writer object to write the Lambda response stream into +public protocol LambdaResponseStreamWriter { + /// Write a response part into the stream. The HTTP response is started lazily before the first call to `write(_:)`. + /// Bytes written to the writer are streamed continually. + func write(_ buffer: ByteBuffer) async throws + /// End the response stream and the underlying HTTP response. + func finish() async throws + /// Write a response part into the stream and end the response stream as well as the underlying HTTP response. + func writeAndFinish(_ buffer: ByteBuffer) async throws +} +``` + +If the user does not call `finish()`, the library will automatically finish the stream after the last `write`. +Appropriate errors will be thrown if `finish()` is called multiple times, or if `write`/`writeAndFinish` is called after +`finish()`. + +### LambdaContext + +`LambdaContext` will be largely unchanged, but the `eventLoop` property will be removed. The `allocator` property of +type `ByteBufferAllocator` will also be removed because (1), we generally want to reduce the number of SwiftNIO types +exposed in the API, and (2), `ByteBufferAllocator` does not optimize the allocation strategies. The common pattern +observed across many libraries is to re-use existing `ByteBuffer`s as much as possible. This is also what we do for the +`LambdaCodableAdapter` (explained in the **Codable Support** section) implementation. + +```swift +/// A context object passed as part of an invocation in LambdaHandler handle functions. +public struct LambdaContext: Sendable { + /// The request ID, which identifies the request that triggered the function invocation. + public var requestID: String { get } + + /// The AWS X-Ray tracing header. + public var traceID: String { get } + + /// The ARN of the Lambda function, version, or alias that's specified in the invocation. + public var invokedFunctionARN: String { get } + + /// The timestamp that the function times out. + public var deadline: DispatchWallTime { get } + + /// For invocations from the AWS Mobile SDK, data about the Amazon Cognito identity provider. + public var cognitoIdentity: String? { get } + + /// For invocations from the AWS Mobile SDK, data about the client application and device. + public var clientContext: String? { get } + + /// `Logger` to log with. + /// + /// - note: The `LogLevel` can be configured using the `LOG_LEVEL` environment variable. + public var logger: Logger { get } +} +``` + +### Handlers + +We introduce three handler protocols: `StreamingLambdaHandler`, `LambdaHandler`, and +`LambdaWithBackgroundProcessingHandler`. + +#### StreamingLambdaHandler + +The new `StreamingLambdaHandler` protocol is the base protocol to implement a Lambda function. Most users will not use +this protocol and instead use the `LambdaHandler` protocol defined below. + +```swift +/// The base StreamingLambdaHandler protocol +public protocol StreamingLambdaHandler { + /// The business logic of the Lambda function + /// - Parameters: + /// - event: The invocation's input data + /// - responseWriter: A ``LambdaResponseStreamWriter`` to write the invocation's response to. + /// If no response or error is written to the `responseWriter` it will + /// report an error to the invoker. + /// - context: The LambdaContext containing the invocation's metadata + /// - Throws: + /// How the thrown error will be handled by the runtime: + /// - An invocation error will be reported if the error is thrown before the first call to + /// ``LambdaResponseStreamWriter.write(_:)``. + /// - If the error is thrown after call(s) to ``LambdaResponseStreamWriter.write(_:)`` but before + /// a call to ``LambdaResponseStreamWriter.finish()``, the response stream will be closed and trailing + /// headers will be sent. + /// - If ``LambdaResponseStreamWriter.finish()`` has already been called before the error is thrown, the + /// error will be logged. + mutating func handle(_ event: ByteBuffer, responseWriter: some LambdaResponseStreamWriter, context: LambdaContext) async throws +} +``` + +Using this protocol requires the `handle` method to receive the incoming event as a `ByteBuffer` and return the output +as a `ByteBuffer` too. + +Through the `LambdaResponseStreamWriter`, which is passed as an argument in the `handle` function, the **response can be +streamed** by calling the `write(_:)` function of the `LambdaResponseStreamWriter` with partial data repeatedly before +finally closing the response stream by calling `finish()`. Users can also choose to return the entire output and not +stream the response by calling `writeAndFinish(_:)`. + +This protocol also allows for background tasks to be run after a result has been reported to the AWS Lambda control +plane, since the `handle(...)` function is free to implement any background work after the call to +`responseWriter.finish()`. + +The protocol is defined in a way that supports a broad range of use-cases. The handle method is marked as `mutating` to +allow handlers to be implemented with a `struct`. + +An implementation that sends the number 1 to 10 every 500ms could look like this: + +```swift +struct SendNumbersWithPause: StreamingLambdaHandler { + func handle( + _ event: ByteBuffer, + responseWriter: some LambdaResponseStreamWriter, + context: LambdaContext + ) async throws { + for i in 1...10 { + // Send partial data + responseWriter.write(ByteBuffer(string: #"\#(i)\n\r"#)) + // Perform some long asynchronous work + try await Task.sleep(for: .milliseconds(500)) + } + // All data has been sent. Close off the response stream. + responseWriter.finish() + } +} +``` + +#### LambdaHandler: + +This handler protocol will be the go-to choice for most use-cases because it is completely agnostic to any +encoding/decoding logic -- conforming objects simply have to implement the `handle` function where the input and return +types are Swift objects. + +Note that the `handle` function does not receive a `LambdaResponseStreamWriter` as an argument. Response streaming is +not viable for `LambdaHandler` because the output has to be encoded prior to it being sent, e.g. it is not possible to +encode a partial/incomplete JSON string. + +```swift +public protocol LambdaHandler { + /// Generic input type + /// The body of the request sent to Lambda will be decoded into this type for the handler to consume + associatedtype Event + /// Generic output type + /// This is the return type of the handle() function. + associatedtype Output + + /// The business logic of the Lambda function. Receives a generic input type and returns a generic output type. + /// Agnostic to encoding/decoding + mutating func handle(_ event: Event, context: LambdaContext) async throws -> Output +} +``` + +#### LambdaWithBackgroundProcessingHandler: + +This protocol is exactly like `LambdaHandler`, with the only difference being the added support for executing background +work after the result has been sent to the AWS Lambda control plane. + +This is achieved by not having a return type in the `handle` function. The output is instead written into a +`LambdaResponseWriter` that is passed in as an argument, meaning that the `handle` function is then free to implement +any background work after the result has been sent to the AWS Lambda control plane. + +`LambdaResponseWriter` has different semantics to the `LambdaResponseStreamWriter`. Where the `write(_:)` function of +`LambdaResponseStreamWriter` means writing into a response stream, the `write(_:)` function of `LambdaResponseWriter` +simply serves as a mechanism to return the output without explicitly returning from the `handle` function. + +```swift +public protocol LambdaResponseWriter { + associatedtype Output + + /// Sends the generic Output object (representing the computed result of the handler) + /// to the AWS Lambda response endpoint. + /// An error will be thrown if this function is called more than once. + func write(_: Output) async throws +} + +public protocol LambdaWithBackgroundProcessingHandler { + /// Generic input type + /// The body of the request sent to Lambda will be decoded into this type for the handler to consume + associatedtype Event + /// Generic output type + /// This is the type that the handle() function will send through the ``LambdaResponseWriter``. + associatedtype Output + + /// The business logic of the Lambda function. Receives a generic input type and returns a generic output type. + /// Agnostic to JSON encoding/decoding + func handle( + _ event: Event, + outputWriter: some LambdaResponseWriter, + context: LambdaContext + ) async throws +} +``` + +##### Example Usage: + +```swift +struct BackgroundProcessingHandler: LambdaWithBackgroundProcessingHandler { + struct Input: Decodable { + let message: String + } + + struct Greeting: Encodable { + let echoedMessage: String + } + + typealias Event = Input + typealias Output = Greeting + + func handle( + _ event: Event, + outputWriter: some LambdaResponseWriter, + context: LambdaContext + ) async throws { + // Return result to the Lambda control plane + try await outputWriter.write(result: Greeting(echoedMessage: event.messageToEcho)) + + // Perform some background work, e.g: + try await Task.sleep(for: .seconds(10)) + + // Exit the function. All asynchronous work has been executed before exiting the scope of this function. + // Follows structured concurrency principles. + return + } +} +``` + +#### Handler Adapters + +Since the `StreamingLambdaHandler` protocol is the base protocol the `LambdaRuntime` works with, there are adapters to +make both `LambdaHandler` and `LambdaWithBackgroundProcessingHandler` compatible with `StreamingLambdaHandler`. + +1. `LambdaHandlerAdapter` accepts a `LambdaHandler` and conforms it to `LambdaWithBackgroundProcessingHandler`. This is + achieved by taking the generic `Output` object returned from the `handle` function of `LambdaHandler` and passing it + to the `write(_:)` function of the `LambdaResponseWriter`. + +2. `LambdaCodableAdapter` accepts a `LambdaWithBackgroundProcessingHandler` and conforms it to `StreamingLambdaHandler`. + This is achieved by wrapping the `LambdaResponseWriter` with the `LambdaResponseStreamWriter` provided by + `StreamingLambdaHandler`. A call to the `write(_:)` function of `LambdaResponseWriter` is translated into a call to + the `writeAndFinish(_:)` function of `LambdaResponseStreamWriter`. + +Both `LambdaHandlerAdapter` and `LambdaCodableAdapter` are described in greater detail in the **Codable Support** +section. + +To summarize, `LambdaHandler` can be used with the `LambdaRuntime` by first going through `LambdaHandlerAdapter` and +then through `LambdaCodableAdapter`. `LambdaWithBackgroundHandler` just requires `LambdaCodableAdapter`. + +For the common JSON-in and JSON-out use-case, there is an extension on `LambdaRuntime` that abstracts away this wrapping +from the user. + +### LambdaRuntime + +`LambdaRuntime` is the class that communicates with the Lambda control plane as defined in +[Building a custom runtime for AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-custom.html) and +forward the invocations to the provided `StreamingLambdaHandler`. It will conform to `ServiceLifecycle.Service` to +provide support for `swift-service-lifecycle`. + +```swift +/// The LambdaRuntime object. This object communicates with the Lambda control plane +/// to fetch work and report errors. +public final class LambdaRuntime: ServiceLifecycle.Service, Sendable + where Handler: StreamingLambdaHandler +{ + + /// Create a LambdaRuntime by passing a handler, an eventLoop and a logger. + /// - Parameter handler: A ``StreamingLambdaHandler`` that will be invoked + /// - Parameter eventLoop: An ``EventLoop`` on which the LambdaRuntime will be + /// executed. Defaults to an EventLoop from + /// ``NIOSingletons.posixEventLoopGroup``. + /// - Parameter logger: A logger + public init( + handler: sending Handler, + eventLoop: EventLoop = Lambda.defaultEventLoop, + logger: Logger = Logger(label: "Lambda") + ) + + /// Create a LambdaRuntime by passing a ``StreamingLambdaHandler``. + public convenience init(handler: sending Handler) + + /// Starts the LambdaRuntime by connecting to the Lambda control plane to ask + /// for events to process. If the environment variable AWS_LAMBDA_RUNTIME_API is + /// set, the LambdaRuntime will connect to the Lambda control plane. Otherwise + /// it will start a mock server that can be used for testing at port 8080 + /// locally. + /// Cancel the task that runs this function to close the communication with + /// the Lambda control plane or close the local mock server. This function + /// only returns once cancelled. + public func run() async throws +} +``` + +The current API allows for a Lambda function to be tested locally through a mock server by requiring an environment +variable named `LOCAL_LAMBDA_SERVER_ENABLED` to be set to `true`. If this environment variable is not set, the program +immediately crashes as the user will not have the `AWS_LAMBDA_RUNTIME_API` environment variable on their local machine +(set automatically when deployed to AWS Lambda). However, making the user set the `LOCAL_LAMBDA_SERVER_ENABLED` +environment variable is an unnecessary step that can be avoided. In the v2 API, the `run()` function will automatically +start the mock server when the `AWS_LAMBDA_RUNTIME_API` environment variable cannot be found. + +### Lambda + +We also add an enum to store a static function and a property on. We put this on the static `Lambda` because +`LambdaRuntime` is generic and thus has bad ergonomics for static properties and functions. + +```swift +enum Lambda { + /// This returns the default EventLoop that a LambdaRuntime is scheduled on. + /// It uses `NIOSingletons.posixEventLoopGroup.next()` under the hood. + public static var defaultEventLoop: any EventLoop { get } + + /// Report a startup error to the Lambda Control Plane API + public static func reportStartupError(any Error) async +} +``` + +Since the library now provides ownership of the `main()` function and allows users to initialize services before the +`LambdaRuntime` is initialized, the library cannot implicitly report +[errors that occur during initialization to the dedicated endpoint AWS exposes](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-api.html#runtimes-api-initerror) +like it currently does through the `initialize()` function of `LambdaRunner` which wraps the handler's `init(...)` and +handles any errors thrown by reporting it to the dedicated AWS endpoint. + +To retain support for initialization error reporting, the `Lambda.reportStartupError(any Error)` function gives users +the option to manually report initialization errors in their closure handler. Although this should ideally happen +implicitly like it currently does in v1, we believe this is a small compromise in comparison to the benefits gained in +now being able to cleanly manage the lifecycles of required services in a structured concurrency manner. + +> Use-case: +> +> Assume we want to load a secret for the Lambda function from a secret vault first. If this fails, we want to report +> the error to the control plane: +> +> ```swift +> let secretVault = SecretVault() +> +> do { +> /// !!! Error thrown: secret "foo" does not exist !!! +> let secret = try await secretVault.getSecret("foo") +> +> let runtime = LambdaRuntime { (event: Input, context: LambdaContext) in +> /// Lambda business logic +> } +> +> let serviceGroup = ServiceGroup( +> services: [postgresClient, runtime], +> configuration: .init(gracefulShutdownSignals: [.sigterm]), +> logger: logger +> ) +> try await serviceGroup.run() +> } catch { +> /// Report startup error straight away to the dedicated initialization error endpoint +> try await Lambda.reportStartupError(error) +> } +> ``` + +### Codable support + +The `LambdaHandler` and `LambdaWithBackgroundProcessingHandler` protocols abstract away encoding/decoding logic from the +conformers as they are generic over custom `Event` and `Output` types. We introduce two adapters `LambdaHandlerAdapter` +and `CodableLambdaAdapter` that implement the encoding/decoding logic and in turn allow the respective handlers to +conform to `StreamingLambdaHandler`. + +#### LambdaHandlerAdapter + +Any handler conforming to `LambdaHandler` can be conformed to `LambdaWithBackgroundProcessingHandler` through +`LambdaHandlerAdapter`. + +```swift +/// Wraps an underlying handler conforming to ``LambdaHandler`` +/// with ``LambdaWithBackgroundProcessingHandler``. +public struct LambdaHandlerAdapter< + Event: Decodable, + Output, + Handler: LambdaHandler +>: LambdaWithBackgroundProcessingHandler where Handler.Event == Event, Handler.Output == Output { + let handler: Handler + + /// Register the concrete handler. + public init(handler: Handler) + + /// 1. Call the `self.handler.handle(...)` with `event` and `context`. + /// 2. Pass the generic `Output` object returned from `self.handler.handle(...)` to `outputWriter.write(_:)` + public func handle(_ event: Event, outputWriter: some LambdaResponseWriter, context: LambdaContext) async throws +} +``` + +#### LambdaCodableAdapter + +`LambdaCodableAdapter` accepts any generic underlying handler conforming to `LambdaWithBackgroundProcessingHandler`. It +also accepts _any_ encoder and decoder object conforming to the `LambdaEventDecoder` and `LambdaOutputEncoder` +protocols: + +##### LambdaEventDecoder and LambdaOutputEncoder protocols + +```swift +public protocol LambdaEventDecoder { + /// Decode the ByteBuffer representing the received event into the generic type Event + /// the handler will receive + func decode(_ type: Event.Type, from buffer: ByteBuffer) throws -> Event +} + +public protocol LambdaOutputEncoder { + /// Encode the generic type Output the handler has produced into a ByteBuffer + func encode(_ value: Output, into buffer: inout ByteBuffer) throws +} +``` + +We provide conformances for Foundation's `JSONDecoder` to `LambdaEventDecoder` and `JSONEncoder` to +`LambdaOutputEncoder`. + +`LambdaCodableAdapter` implements its `handle()` method by: + +1. Decoding the `ByteBuffer` event into the generic `Event` type. +2. Wrapping the `LambdaResponseStreamWriter` with a concrete `LambdaResponseWriter` such that calls to + `LambdaResponseWriter`s `write(_:)` are mapped to `LambdaResponseStreamWriter`s `writeAndFinish(_:)`. + - Note that the argument to `LambdaResponseWriter`s `write(_:)` is a generic `Output` object whereas + `LambdaResponseStreamWriter`s `writeAndFinish(_:)` requires a `ByteBuffer`. + - Therefore, the concrete implementation of `LambdaResponseWriter` also accepts an encoder. Its `write(_:)` function + first encodes the generic `Output` object and then passes it to the underlying `LambdaResponseStreamWriter`. +3. Passing the generic `Event` instance, the concrete `LambdaResponseWriter`, as well as the `LambdaContext` to the + underlying handler's `handle()` method. + +`LambdaCodableAdapter` can implement encoding/decoding for _any_ handler conforming to +`LambdaWithBackgroundProcessingHandler` if `Event` is `Decodable` and the `Output` is `Encodable` or `Void`, meaning +that the encoding/decoding stubs do not need to be implemented by the user. + +```swift +/// Wraps an underlying handler conforming to `LambdaWithBackgroundProcessingHandler` +/// with encoding/decoding logic +public struct LambdaCodableAdapter< + Handler: LambdaWithBackgroundProcessingHandler, + Event: Decodable, + Output, + Decoder: LambdaEventDecoder, + Encoder: LambdaOutputEncoder +>: StreamingLambdaHandler where Handler.Output == Output, Handler.Event == Event { + + /// Register the concrete handler, encoder, and decoder. + public init( + handler: Handler, + encoder: Encoder, + decoder: Decoder + ) where Output: Encodable + + /// For handler with a void output -- the user doesn't specify an encoder. + public init( + handler: Handler, + decoder: Decoder + ) where Output == Void, Encoder == VoidEncoder + + /// 1. Decode the invocation event using `self.decoder` + /// 2. Create a concrete `LambdaResponseWriter` that maps calls to `write(_:)` with the `responseWriter`s `writeAndFinish(_:)` + /// 2. Call the underlying `self.handler.handle()` method with the decoded event data, the concrete `LambdaResponseWriter`, + /// and the `LambdaContext`. + public mutating func handle( + _ request: ByteBuffer, + responseWriter: some LambdaResponseStreamWriter, + context: LambdaContext + ) async throws +} +``` + +### Handler as a Closure + +To create a Lambda function using the current API, a user first has to create an object and conform it to one of the +handler protocols by implementing the initializer and the `handle(...)` function. Now that `LambdaRuntime` is public, +this verbosity can very easily be simplified. + +#### ClosureHandler + +This handler is generic over any `Event` type conforming to `Decodable` and any `Output` type conforming to `Encodable` +or `Void`. + +```swift +public struct ClosureHandler: LambdaHandler { + /// Initialize with a closure handler over generic Input and Output types + public init(body: @escaping (Event, LambdaContext) async throws -> Output) where Output: Encodable + /// Initialize with a closure handler over a generic Input type (Void Output). + public init(body: @escaping (Event, LambdaContext) async throws -> Void) where Output == Void + /// The business logic of the Lambda function. + public func handle(_ event: Event, context: LambdaContext) async throws -> Output +} +``` + +Given that `ClosureHandler` conforms to `LambdaHandler`: + +1. We can extend the `LambdaRuntime` initializer such that it accepts a closure as an argument. +2. Within the initializer, the closure handler is wrapped with `LambdaCodableAdapter`. + +```swift +extension LambdaRuntime { + /// Initialize a LambdaRuntime with a closure handler over generic Event and Output types. + /// This initializer bolts on encoding/decoding logic by wrapping the closure handler with + /// LambdaCodableAdapter. + public init( + body: @escaping (Event, LambdaContext) async throws -> Output + ) where Handler == LambdaCodableAdapter, Event, Output, JSONDecoder, JSONEncoder> + + /// Same as above but for handlers with a void output + public init( + body: @escaping (Event, LambdaContext) async throws -> Void + ) where Handler == LambdaCodableAdapter, Event, Void, JSONDecoder, VoidEncoder> +} +``` + +We can now significantly reduce the verbosity and leverage Swift's trailing closure syntax to cleanly create and run a +Lambda function, abstracting away the decoding and encoding logic from the user: + +```swift +/// The type the handler will use as input +struct Input: Decodable { + var message: String +} + +/// The type the handler will output +struct Greeting: Encodable { + var echoedMessage: String +} + +/// A simple Lambda function that echoes the input +let runtime = LambdaRuntime { (event: Input, context: LambdaContext) in + Greeting(echoedMessage: event.message) +} + +try await runtime.run() +``` + +We also add a `StreamingClosureHandler` conforming to `StreamingLambdaHandler` for use-cases where the user wants to +handle encoding/decoding themselves: + +```swift +public struct StreamingClosureHandler: StreamingLambdaHandler { + + public init( + body: @escaping sending (ByteBuffer, LambdaResponseStreamWriter, LambdaContext) async throws -> () + ) + + public func handle( + _ request: ByteBuffer, + responseWriter: LambdaResponseStreamWriter, + context: LambdaContext + ) async throws +} + +extension LambdaRuntime { + public init( + body: @escaping sending (ByteBuffer, LambdaResponseStreamWriter, LambdaContext) async throws -> () + ) +} +``` + +## Alternatives considered + +### [UInt8] instead of ByteBuffer + +We considered using `[UInt8]` instead of `ByteBuffer` in the base `LambdaHandler` API. We decided to use `ByteBuffer` +for two reasons. + +1. 99% of use-cases will use the JSON codable API and will not directly get in touch with ByteBuffer anyway. For those + users it does not matter if the base API uses `ByteBuffer` or `[UInt8]`. +2. The incoming and outgoing data must be in the `ByteBuffer` format anyway, as Lambda uses SwiftNIO under the hood and + SwiftNIO uses `ByteBuffer` in its APIs. By using `ByteBuffer` we can save a copies to and from `[UInt8]`. This will + reduce the invocation time for all users. +3. The base `LambdaHandler` API is most likely mainly being used by developers that want to integrate their web + framework with Lambda (examples: Vapor, Hummingbird, ...). Those developers will most likely prefer to get the data + in the `ByteBuffer` format anyway, as their lower level networking stack also depends on SwiftNIO. + +### Users create a LambdaResponse, that supports streaming instead of being passed a LambdaResponseStreamWriter + +Instead of passing the `LambdaResponseStreamWriter` in the invocation we considered a new type `LambdaResponse`, that +users must return in the `StreamingLambdaHandler`. + +Its API would look like this: + +```swift +/// A response returned from a ``LambdaHandler``. +/// The response can be empty, a single ByteBuffer or a response stream. +public struct LambdaResponse { + /// A writer to be used when creating a streamed response. + public struct Writer { + /// Writes data to the response stream + public func write(_ byteBuffer: ByteBuffer) async throws + /// Closes off the response stream + public func finish() async throws + /// Writes the `byteBuffer` to the response stream and subsequently closes the stream + public func writeAndFinish(_ byteBuffer: ByteBuffer) async throws + } + + /// Creates an empty lambda response + public init() + + /// Creates a LambdaResponse with a fixed ByteBuffer. + public init(_ byteBuffer: ByteBuffer) + + /// Creates a streamed lambda response. Use the ``Writer`` to send + /// response chunks on the stream. + public init(_ stream: @escaping sending (Writer) async throws -> ()) +} +``` + +The `StreamingLambdaHandler` would look like this: + +```swift +/// The base LambdaHandler protocol +public protocol StreamingLambdaHandler { + /// The business logic of the Lambda function + /// - Parameters: + /// - event: The invocation's input data + /// - context: The LambdaContext containing the invocation's metadata + /// - Returns: A LambdaResponse, that can be streamed + mutating func handle( + _ event: ByteBuffer, + context: LambdaContext + ) async throws -> LambdaResponse +} +``` + +There are pros and cons for the API that returns the `LambdaResponses` and there are pros and cons for the API that +receives a `LambdaResponseStreamWriter` as a parameter. + +Concerning following structured concurrency principles the approach that receives a `LambdaResponseStreamWriter` as a +parameter has benefits as the lifetime of the handle function is tied to the invocation runtime. The approach that +returns a `LambdaResponse` splits the invocation into two separate function calls. First the handle method is invoked, +second the `LambdaResponse` writer closure is invoked. This means that it is impossible to use Swift APIs that use +`with` style lifecycle management patterns from before creating the response until sending the full response stream off. +For example, users instrumenting their lambdas with Swift tracing likely can not use the `withSpan` API for the full +lifetime of the request, if they return a streamed response. + +However, if it comes to consistency with the larger Swift on server ecosystem, the API that returns a `LambdaResponse` +is likely the better choice. Hummingbird v2, OpenAPI and the new Swift gRPC v2 implementation all use this approach. +This might be due to the fact that writing middleware becomes easier, if a Response is explicitly returned. + +We decided to implement the approach in which a `LambdaResponseStreamWriter` is passed to the function, since the +approach in which a `LambdaResponse` is returned can trivially be built on top of it. This is not true vice versa. + +We welcome the discussion on this topic and are open to change our minds and API here. + +### Adding a function `addBackgroundTask(_ body: sending @escaping () async -> ())` in `LambdaContext` + +Initially we proposed an explicit `addBackgroundTask(_:)` function in `LambdaContext` that users could call from their +handler object to schedule a background task to be run after the result is reported to AWS. We received feedback that +this approach for supporting background tasks does not exhibit structured concurrency, as code could still be in +execution after leaving the scope of the `handle(...)` function. + +For handlers conforming to the `StreamingLambdaHandler`, `addBackgroundTask(_:)` was anyways unnecessary as background +work could be executed in a structured concurrency manner within the `handle(...)` function after the call to +`LambdaResponseStreamWriter.finish()`. + +For handlers conforming to the `LambdaHandler` protocol, we considered extending `LambdaHandler` with a +`performPostHandleWork(...)` function that will be called after the `handle` function by the library. Users wishing to +add background work can override this function in their `LambdaHandler` conforming object. + +```swift +public protocol LambdaHandler { + associatedtype Event + associatedtype Output + + func handle(_ event: Event, context: LambdaContext) async throws -> Output + + func performPostHandleWork(...) async throws -> Void +} + +extension LambdaHandler { + // User's can override this function if they wish to perform background work + // after returning a response from ``handle``. + func performPostHandleWork(...) async throws -> Void { + // nothing to do + } +} +``` + +Yet this poses difficulties when the user wishes to use any state created in the `handle(...)` function as part of the +background work. + +In general, the most common use-case for this library will be to implement simple Lambda functions that do not have +requirements for response streaming, nor to perform any background work after returning the output. To keep things easy +for the common use-case, and with Swift's principle of progressive disclosure of complexity in mind, we settled on three +handler protocols: + +1. `LambdaHandler`: Most common use-case. JSON-in, JSON-out. Does not support background work execution. An intuitive + `handle(event: Event, context: LambdaContext) -> Output` API that is simple to understand, i.e. users are not exposed + to the concept of sending their response through a writer. `LambdaHandler` can be very cleanly implemented and used + with `LambdaRuntime`, especially with `ClosureHandler`. +2. `LambdaWithBackgroundProcessingHandler`: If users wish to augment their `LambdaHandler` with the ability to run + background tasks, they can easily migrate. A user simply has to: + 1. Change the conformance to `LambdaWithBackgroundProcessingHandler`. + 2. Add an additional `outputWriter: some LambdaResponseWriter` argument to the `handle` function. + 3. Replace the `return ...` with `outputWriter.write(...)`. + 4. Implement any background work after `outputWriter.write(...)`. +3. `StreamingLambdaHandler`: This is the base handler protocol which is intended to be used directly only for advanced + use-cases. Users are provided the invocation event as a `ByteBuffer` and a `LambdaResponseStreamWriter` where the + computed result (as `ByteBuffer`) can either be streamed (with repeated calls to `write(_:)`) or sent all at once + (with a single call to `writeAndFinish(_:)`). After closing the `LambdaResponseStreamWriter`, any background work can + be implemented. + +### Making LambdaResponseStreamWriter and LambdaResponseWriter ~Copyable + +We initially proposed to make the `LambdaResponseStreamWriter` and `LambdaResponseWriter` protocols `~Copyable`, with +the functions that close the response having the `consuming` ownership keyword. This was so that the compiler could +enforce the restriction of not being able to interact with the writer after the response stream has closed. + +However, non-copyable types do not compose nicely and add complexity for users. Further, for the compiler to actually +enforce the `consuming` restrictions, user's have to explicitly mark the writer argument as `consuming` in the `handle` +function. + +Therefore, throwing appropriate errors to prevent abnormal interaction with the writers seems to be the simplest +approach. + +## A word about versioning + +We are aware that AWS Lambda Runtime has not reached a proper 1.0. We intend to keep the current implementation around +at 1.0-alpha. We don't want to change the current API without releasing a new major. We think there are lots of adopters +out there that depend on the API in v1. Because of this we intend to release the proposed API here as AWS Lambda Runtime +v2. From b30dfe434dfa05c75178c732e0db8f00e880cb62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 11:05:27 +0200 Subject: [PATCH 11/25] fix variable name for matrix action --- .github/workflows/pull_request.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9bc435c9..9633cdb8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -26,14 +26,13 @@ jobs: integration-tests: name: Integration Tests - # Workaround https://github.com/nektos/act/issues/1875 uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main with: name: "Integration tests" matrix_linux_command: "./scripts/integration_tests.sh" - linux_5_8_enabled: false - linux_5_9_enabled: false - linux_5_10_enabled: false + matrix_linux_5_8_enabled: false + matrix_linux_5_9_enabled: false + matrix_linux_5_10_enabled: false swift-6-language-mode: name: Swift 6 Language Mode From ba369073796c98a48f1d85ffd64f2ee1121ab04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 14:58:45 +0200 Subject: [PATCH 12/25] first test to get a matrix setup --- .github/workflows/examples_matrix.yml | 80 +++++++++++++++++++++++++++ .github/workflows/pull_request.yml | 7 +-- scripts/integration_tests.sh | 42 +++++++++++--- 3 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/examples_matrix.yml diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml new file mode 100644 index 00000000..aa338e8a --- /dev/null +++ b/.github/workflows/examples_matrix.yml @@ -0,0 +1,80 @@ +name: ExamplesMatrix + +on: + workflow_call: + inputs: + name: + type: string + description: "The name of the workflow used for the concurrency group." + required: true + matrix_linux_command: + type: string + description: "The command of the current Swift version linux matrix job to execute." + required: true + matrix_linux_nightly_6_0_enabled: + type: boolean + description: "Boolean to enable the nightly 6.0 Swift version matrix job. Defaults to true." + default: true + matrix_linux_nightly_6_0_container_image: + type: string + description: "Container image for the nightly 6.0 Swift version matrix job. Defaults to matching Swift Ubuntu image." + default: "swiftlang/swift:nightly-6.0-jammy" + matrix_linux_nightly_6_0_command_override: + type: string + description: "The command of the nightly 6.0 Swift version linux matrix job to execute." + matrix_linux_nightly_main_enabled: + type: boolean + description: "Boolean to enable the nightly main Swift version matrix job. Defaults to true." + default: true + matrix_linux_nightly_main_container_image: + type: string + description: "Container image for the nightly main Swift version matrix job. Defaults to matching Swift Ubuntu image." + default: "swiftlang/swift:nightly-main-jammy" + matrix_linux_nightly_main_command_override: + type: string + description: "The command of the nightly main Swift version linux matrix job to execute." + +## We are cancelling previously triggered workflow runs +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.name }} + cancel-in-progress: true + +jobs: + linux: + name: Linux (${{ matrix.swift.swift_version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + examples : [ "HelloWorld", "APIGateway" ] + # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release + swift: + - image: ${{ inputs.matrix_linux_nightly_6_0_container_image }} + swift_version: "nightly-6.0" + enabled: ${{ inputs.matrix_linux_nightly_6_0_enabled }} + - image: ${{ inputs.matrix_linux_nightly_main_container_image }} + swift_version: "nightly-main" + enabled: ${{ inputs.matrix_linux_nightly_main_enabled }} + container: + image: ${{ matrix.swift.image }} + steps: + - name: Checkout repository + if: ${{ matrix.swift.enabled }} + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Mark the workspace as safe + if: ${{ matrix.swift.enabled }} + # https://github.com/actions/checkout/issues/766 + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Run matrix job + if: ${{ matrix.swift.enabled }} + env: + SWIFT_VERSION: ${{ matrix.swift.swift_version }} + COMMAND: ${{ inputs.matrix_linux_command }} + COMMAND_OVERRIDE_NIGHTLY_6_0: ${{ inputs.matrix_linux_nightly_6_0_command_override }} + COMMAND_OVERRIDE_NIGHTLY_MAIN: ${{ inputs.matrix_linux_nightly_main_command_override }} + EXAMPLE: ${{ matrix.examples }} + run: | + apt-get -qq update && apt-get -qq -y install curl + ./scripts/integration_tests.sh \ No newline at end of file diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9633cdb8..c6f9e938 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -26,13 +26,10 @@ jobs: integration-tests: name: Integration Tests - uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main + uses: examples_matrix.yml with: name: "Integration tests" - matrix_linux_command: "./scripts/integration_tests.sh" - matrix_linux_5_8_enabled: false - matrix_linux_5_9_enabled: false - matrix_linux_5_10_enabled: false + matrix_linux_command: "pushd Examples/$EXAMPLE && LAMBDA_USE_LOCAL_DEPS=../.. swift build && popd" swift-6-language-mode: name: Swift 6 Language Mode diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh index 816d666e..d8b9363c 100755 --- a/scripts/integration_tests.sh +++ b/scripts/integration_tests.sh @@ -13,12 +13,38 @@ ## ##===----------------------------------------------------------------------===## -set +x -e +# set +x -e -for EXAMPLE in $(find Examples -type d -d 1); -do - echo "Building $EXAMPLE" - pushd $EXAMPLE - LAMBDA_USE_LOCAL_DEPS=../.. swift build - popd -done +# for EXAMPLE in $(find Examples -type d -d 1); +# do +# echo "Building $EXAMPLE" +# pushd $EXAMPLE +# LAMBDA_USE_LOCAL_DEPS=../.. swift build +# popd +# done + +set -euo pipefail + +log() { printf -- "** %s\n" "$*" >&2; } +error() { printf -- "** ERROR: %s\n" "$*" >&2; } +fatal() { error "$@"; exit 1; } + +test -n "${SWIFT_VERSION:-}" || fatal "SWIFT_VERSION unset" +test -n "${COMMAND:-}" || fatal "COMMAND unset" +test -n "${EXAMPLE:-}" || fatal "EXAMPLE unset" +swift_version="$SWIFT_VERSION" +command="$COMMAND" +command_nightly_6_0="$COMMAND_OVERRIDE_NIGHTLY_6_0" +command_nightly_main="$COMMAND_OVERRIDE_NIGHTLY_MAIN" +example="$EXAMPLE" + +if [[ "$swift_version" == "nightly-6.0" ]] && [[ -n "$command_nightly_6_0" ]]; then + log "Running nightly 6.0 command override" + eval "$command_nightly_6_0" +elif [[ "$swift_version" == "nightly-main" ]] && [[ -n "$command_nightly_main" ]]; then + log "Running nightly main command override" + eval "$command_nightly_main" +else + log "Running default command" + eval "$command" +fi From 876b237140618c34e808de8de8514ba0faf447cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 15:05:45 +0200 Subject: [PATCH 13/25] refine matrix --- .github/workflows/pull_request.yml | 2 +- scripts/integration_tests.sh | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c6f9e938..604adece 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -29,7 +29,7 @@ jobs: uses: examples_matrix.yml with: name: "Integration tests" - matrix_linux_command: "pushd Examples/$EXAMPLE && LAMBDA_USE_LOCAL_DEPS=../.. swift build && popd" + matrix_linux_command: "LAMBDA_USE_LOCAL_DEPS=../.. swift build" swift-6-language-mode: name: Swift 6 Language Mode diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh index d8b9363c..5d209d46 100755 --- a/scripts/integration_tests.sh +++ b/scripts/integration_tests.sh @@ -13,16 +13,6 @@ ## ##===----------------------------------------------------------------------===## -# set +x -e - -# for EXAMPLE in $(find Examples -type d -d 1); -# do -# echo "Building $EXAMPLE" -# pushd $EXAMPLE -# LAMBDA_USE_LOCAL_DEPS=../.. swift build -# popd -# done - set -euo pipefail log() { printf -- "** %s\n" "$*" >&2; } @@ -38,6 +28,8 @@ command_nightly_6_0="$COMMAND_OVERRIDE_NIGHTLY_6_0" command_nightly_main="$COMMAND_OVERRIDE_NIGHTLY_MAIN" example="$EXAMPLE" +pushd Examples/"$example" > /dev/null + if [[ "$swift_version" == "nightly-6.0" ]] && [[ -n "$command_nightly_6_0" ]]; then log "Running nightly 6.0 command override" eval "$command_nightly_6_0" @@ -48,3 +40,5 @@ else log "Running default command" eval "$command" fi + +popd From 534502be27294628830e443e7113e7266377c0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 15:07:42 +0200 Subject: [PATCH 14/25] refine matrix --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 604adece..92933e07 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -26,7 +26,7 @@ jobs: integration-tests: name: Integration Tests - uses: examples_matrix.yml + uses: swift-server/swift-aws-lambda-runtime/.github/workflows/examples_matrix.yml with: name: "Integration tests" matrix_linux_command: "LAMBDA_USE_LOCAL_DEPS=../.. swift build" From dd21f48066981ad734ea65c9f2741f18e7bbe587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 15:11:51 +0200 Subject: [PATCH 15/25] fix reference to sub workflow --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 92933e07..c6decf1e 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -26,7 +26,7 @@ jobs: integration-tests: name: Integration Tests - uses: swift-server/swift-aws-lambda-runtime/.github/workflows/examples_matrix.yml + uses: ./.github/workflows/examples_matrix.yml with: name: "Integration tests" matrix_linux_command: "LAMBDA_USE_LOCAL_DEPS=../.. swift build" From cce5208c1f08e49b342f1ea985b0dbdf5bce884e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 15:22:11 +0200 Subject: [PATCH 16/25] remove references from docker --- readme.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/readme.md b/readme.md index 1de0000e..e33b5379 100644 --- a/readme.md +++ b/readme.md @@ -163,11 +163,3 @@ tbd + link to docc ### Background Tasks tbd + link to docc - -### Development - -Your contributions are welcome. If you open a pull request, be sure to ensure it passes the soundness checks - -``` -docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.al2.6.0.yaml run soundness -``` From 64337df924691c530f2728142752294ed40a0649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 15:25:24 +0200 Subject: [PATCH 17/25] test passing an array --- .github/workflows/examples_matrix.yml | 11 ++++++++++- .github/workflows/pull_request.yml | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index aa338e8a..69d031b3 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -7,6 +7,13 @@ on: type: string description: "The name of the workflow used for the concurrency group." required: true + examples: + type: string + description: "The examples to run." + required: true + matrix_linux_command_override: + type: string + description: "The command of the current Swift version linux matrix job to execute." matrix_linux_command: type: string description: "The command of the current Swift version linux matrix job to execute." @@ -46,7 +53,9 @@ jobs: strategy: fail-fast: false matrix: - examples : [ "HelloWorld", "APIGateway" ] + # This should be passed as an argument in input. Can we pass arrays as argument ? + # examples : [ "HelloWorld", "APIGateway" ] + examples: ${{ inputs.examples }} # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release swift: - image: ${{ inputs.matrix_linux_nightly_6_0_container_image }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c6decf1e..32a5675b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -28,6 +28,8 @@ jobs: name: Integration Tests uses: ./.github/workflows/examples_matrix.yml with: + # We should pass the list of examples here, but we can't pass an array as argument + examples: [ "HelloWorld", "APIGateway" ] name: "Integration tests" matrix_linux_command: "LAMBDA_USE_LOCAL_DEPS=../.. swift build" From 7df6a049f988f958daa2c38203d6420ec77a8895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 15:26:27 +0200 Subject: [PATCH 18/25] try to pass an array --- .github/workflows/examples_matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index 69d031b3..9bf47edf 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: name: - type: string + type: sequence description: "The name of the workflow used for the concurrency group." required: true examples: From 26c6e31fa50e5efb7665df3f1887482c502252d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 15:27:08 +0200 Subject: [PATCH 19/25] test passing array --- .github/workflows/examples_matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index 9bf47edf..c229f847 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -4,11 +4,11 @@ on: workflow_call: inputs: name: - type: sequence + type: string description: "The name of the workflow used for the concurrency group." required: true examples: - type: string + type: sequence description: "The examples to run." required: true matrix_linux_command_override: From 6156e93469a994b5c393b485c158439c32221a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 15:28:02 +0200 Subject: [PATCH 20/25] revert back to hard coded example list in the matrix --- .github/workflows/examples_matrix.yml | 13 +++++++------ .github/workflows/pull_request.yml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index c229f847..1ba4fdaf 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -7,10 +7,10 @@ on: type: string description: "The name of the workflow used for the concurrency group." required: true - examples: - type: sequence - description: "The examples to run." - required: true + # examples: + # type: sequence + # description: "The examples to run." + # required: true matrix_linux_command_override: type: string description: "The command of the current Swift version linux matrix job to execute." @@ -54,8 +54,9 @@ jobs: fail-fast: false matrix: # This should be passed as an argument in input. Can we pass arrays as argument ? - # examples : [ "HelloWorld", "APIGateway" ] - examples: ${{ inputs.examples }} + examples : [ "HelloWorld", "APIGateway" ] + # examples: ${{ inputs.examples }} + # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release swift: - image: ${{ inputs.matrix_linux_nightly_6_0_container_image }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 32a5675b..77f9a5af 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -29,7 +29,7 @@ jobs: uses: ./.github/workflows/examples_matrix.yml with: # We should pass the list of examples here, but we can't pass an array as argument - examples: [ "HelloWorld", "APIGateway" ] + # examples: [ "HelloWorld", "APIGateway" ] name: "Integration tests" matrix_linux_command: "LAMBDA_USE_LOCAL_DEPS=../.. swift build" From c8ba6009497b1d69a8cc85fe25a11f8355ed2e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 21:57:49 +0200 Subject: [PATCH 21/25] add the example name in the job name --- .github/workflows/examples_matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index 1ba4fdaf..7fb358ff 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -48,7 +48,7 @@ concurrency: jobs: linux: - name: Linux (${{ matrix.swift.swift_version }}) + name: example-${{ matrix.examples }}-linux-${{ matrix.swift.swift_version }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -56,7 +56,7 @@ jobs: # This should be passed as an argument in input. Can we pass arrays as argument ? examples : [ "HelloWorld", "APIGateway" ] # examples: ${{ inputs.examples }} - + # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release swift: - image: ${{ inputs.matrix_linux_nightly_6_0_container_image }} From dd84d7cd53ff42cc0f099ef7606c7086da56a127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 22:01:28 +0200 Subject: [PATCH 22/25] improve naming --- .github/workflows/examples_matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index 7fb358ff..580f80fb 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -48,7 +48,7 @@ concurrency: jobs: linux: - name: example-${{ matrix.examples }}-linux-${{ matrix.swift.swift_version }} + name: Example/${{ matrix.examples }} on Linux ${{ matrix.swift.swift_version }} runs-on: ubuntu-latest strategy: fail-fast: false From 639224f5ef34e1e5096f5d6e6283b42e44aef2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 22:13:15 +0200 Subject: [PATCH 23/25] Use only one Swift version to build the examples --- .github/workflows/examples_matrix.yml | 42 ++++----------------------- scripts/integration_tests.sh | 14 ++------- 2 files changed, 8 insertions(+), 48 deletions(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index 580f80fb..bab64097 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -11,35 +11,14 @@ on: # type: sequence # description: "The examples to run." # required: true - matrix_linux_command_override: - type: string - description: "The command of the current Swift version linux matrix job to execute." matrix_linux_command: type: string description: "The command of the current Swift version linux matrix job to execute." required: true - matrix_linux_nightly_6_0_enabled: - type: boolean - description: "Boolean to enable the nightly 6.0 Swift version matrix job. Defaults to true." - default: true - matrix_linux_nightly_6_0_container_image: - type: string - description: "Container image for the nightly 6.0 Swift version matrix job. Defaults to matching Swift Ubuntu image." - default: "swiftlang/swift:nightly-6.0-jammy" - matrix_linux_nightly_6_0_command_override: - type: string - description: "The command of the nightly 6.0 Swift version linux matrix job to execute." - matrix_linux_nightly_main_enabled: - type: boolean - description: "Boolean to enable the nightly main Swift version matrix job. Defaults to true." - default: true - matrix_linux_nightly_main_container_image: - type: string - description: "Container image for the nightly main Swift version matrix job. Defaults to matching Swift Ubuntu image." - default: "swiftlang/swift:nightly-main-jammy" - matrix_linux_nightly_main_command_override: + matrix_linux_swift_container_image: type: string - description: "The command of the nightly main Swift version linux matrix job to execute." + description: "Container image for the 6.0 Swift version matrix job. Defaults to matching latest Swift Ubuntu image." + default: "swiftlang/swift:6.0.1-noble" ## We are cancelling previously triggered workflow runs concurrency: @@ -57,33 +36,24 @@ jobs: examples : [ "HelloWorld", "APIGateway" ] # examples: ${{ inputs.examples }} - # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release + # We are using only one Swift version swift: - - image: ${{ inputs.matrix_linux_nightly_6_0_container_image }} - swift_version: "nightly-6.0" - enabled: ${{ inputs.matrix_linux_nightly_6_0_enabled }} - - image: ${{ inputs.matrix_linux_nightly_main_container_image }} - swift_version: "nightly-main" - enabled: ${{ inputs.matrix_linux_nightly_main_enabled }} + - image: ${{ inputs.matrix_linux_swift_container_image }} + swift_version: "6.0.1-noble" container: image: ${{ matrix.swift.image }} steps: - name: Checkout repository - if: ${{ matrix.swift.enabled }} uses: actions/checkout@v4 with: persist-credentials: false - name: Mark the workspace as safe - if: ${{ matrix.swift.enabled }} # https://github.com/actions/checkout/issues/766 run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Run matrix job - if: ${{ matrix.swift.enabled }} env: SWIFT_VERSION: ${{ matrix.swift.swift_version }} COMMAND: ${{ inputs.matrix_linux_command }} - COMMAND_OVERRIDE_NIGHTLY_6_0: ${{ inputs.matrix_linux_nightly_6_0_command_override }} - COMMAND_OVERRIDE_NIGHTLY_MAIN: ${{ inputs.matrix_linux_nightly_main_command_override }} EXAMPLE: ${{ matrix.examples }} run: | apt-get -qq update && apt-get -qq -y install curl diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh index 5d209d46..232d743d 100755 --- a/scripts/integration_tests.sh +++ b/scripts/integration_tests.sh @@ -24,21 +24,11 @@ test -n "${COMMAND:-}" || fatal "COMMAND unset" test -n "${EXAMPLE:-}" || fatal "EXAMPLE unset" swift_version="$SWIFT_VERSION" command="$COMMAND" -command_nightly_6_0="$COMMAND_OVERRIDE_NIGHTLY_6_0" -command_nightly_main="$COMMAND_OVERRIDE_NIGHTLY_MAIN" example="$EXAMPLE" pushd Examples/"$example" > /dev/null -if [[ "$swift_version" == "nightly-6.0" ]] && [[ -n "$command_nightly_6_0" ]]; then - log "Running nightly 6.0 command override" - eval "$command_nightly_6_0" -elif [[ "$swift_version" == "nightly-main" ]] && [[ -n "$command_nightly_main" ]]; then - log "Running nightly main command override" - eval "$command_nightly_main" -else - log "Running default command" - eval "$command" -fi +log "Running command with Swift $SWIFT_VERSION" +eval "$command" popd From 9c89ff27cb5e40e98f5a5025f00d17c597ad70a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 22:22:39 +0200 Subject: [PATCH 24/25] fix image name --- .github/workflows/examples_matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index bab64097..0032e402 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -18,7 +18,7 @@ on: matrix_linux_swift_container_image: type: string description: "Container image for the 6.0 Swift version matrix job. Defaults to matching latest Swift Ubuntu image." - default: "swiftlang/swift:6.0.1-noble" + default: "swift:6.0.1-noble" ## We are cancelling previously triggered workflow runs concurrency: From 0f5ce60448cef5bab203ee2ca2e63681872e55e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Wed, 9 Oct 2024 22:27:31 +0200 Subject: [PATCH 25/25] use latest alias to fetch the latest version --- .github/workflows/examples_matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples_matrix.yml b/.github/workflows/examples_matrix.yml index 0032e402..39ec1d26 100644 --- a/.github/workflows/examples_matrix.yml +++ b/.github/workflows/examples_matrix.yml @@ -17,8 +17,8 @@ on: required: true matrix_linux_swift_container_image: type: string - description: "Container image for the 6.0 Swift version matrix job. Defaults to matching latest Swift Ubuntu image." - default: "swift:6.0.1-noble" + description: "Container image for the matrix job. Defaults to matching latest Swift Ubuntu image." + default: "swift:latest" ## We are cancelling previously triggered workflow runs concurrency: