From 378cc0512223f9adfe6af0d7ca1b9ca8e6b8b5ba Mon Sep 17 00:00:00 2001 From: Maxime David Date: Tue, 18 Jun 2024 10:49:49 +0100 Subject: [PATCH 01/14] build: add dependabot config for github actions (#52) * add dependabot config for github actions --- .github/dependabot.yml | 7 +++++++ .github/workflows/aws-lambda-java-core.yml | 5 +++-- .../workflows/aws-lambda-java-events-sdk-transformer.yml | 5 +++-- .github/workflows/aws-lambda-java-events.yml | 5 +++-- .github/workflows/aws-lambda-java-log4j2.yml | 5 +++-- .github/workflows/aws-lambda-java-serialization.yml | 5 +++-- .github/workflows/aws-lambda-java-tests.yml | 5 +++-- .github/workflows/repo-sync.yml | 4 ++++ .github/workflows/runtime-interface-client_pr.yml | 3 ++- .github/workflows/samples.yml | 1 + 10 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..3722537a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" \ No newline at end of file diff --git a/.github/workflows/aws-lambda-java-core.yml b/.github/workflows/aws-lambda-java-core.yml index 0b553bbc..1ba1ed2c 100644 --- a/.github/workflows/aws-lambda-java-core.yml +++ b/.github/workflows/aws-lambda-java-core.yml @@ -7,11 +7,12 @@ on: push: branches: [ main ] paths: - - 'aws-lambda-java-core/**' + - 'aws-lambda-java-core/**' pull_request: branches: [ '*' ] paths: - - 'aws-lambda-java-core/**' + - 'aws-lambda-java-core/**' + - '.github/workflows/aws-lambda-java-core.yml' jobs: build: diff --git a/.github/workflows/aws-lambda-java-events-sdk-transformer.yml b/.github/workflows/aws-lambda-java-events-sdk-transformer.yml index 679639d3..7749fedc 100644 --- a/.github/workflows/aws-lambda-java-events-sdk-transformer.yml +++ b/.github/workflows/aws-lambda-java-events-sdk-transformer.yml @@ -7,11 +7,12 @@ on: push: branches: [ main ] paths: - - 'aws-lambda-java-events-sdk-transformer/**' + - 'aws-lambda-java-events-sdk-transformer/**' pull_request: branches: [ '*' ] paths: - - 'aws-lambda-java-events-sdk-transformer/**' + - 'aws-lambda-java-events-sdk-transformer/**' + - '.github/workflows/aws-lambda-java-events-sdk-transformer.yml' jobs: build: diff --git a/.github/workflows/aws-lambda-java-events.yml b/.github/workflows/aws-lambda-java-events.yml index bdd01eb7..7caf8164 100644 --- a/.github/workflows/aws-lambda-java-events.yml +++ b/.github/workflows/aws-lambda-java-events.yml @@ -7,11 +7,12 @@ on: push: branches: [ main ] paths: - - 'aws-lambda-java-events/**' + - 'aws-lambda-java-events/**' pull_request: branches: [ '*' ] paths: - - 'aws-lambda-java-events/**' + - 'aws-lambda-java-events/**' + - '.github/workflows/aws-lambda-java-events.yml' jobs: build: diff --git a/.github/workflows/aws-lambda-java-log4j2.yml b/.github/workflows/aws-lambda-java-log4j2.yml index 427c7536..edda4334 100644 --- a/.github/workflows/aws-lambda-java-log4j2.yml +++ b/.github/workflows/aws-lambda-java-log4j2.yml @@ -7,11 +7,12 @@ on: push: branches: [ main ] paths: - - 'aws-lambda-java-log4j2/**' + - 'aws-lambda-java-log4j2/**' pull_request: branches: [ '*' ] paths: - - 'aws-lambda-java-log4j2/**' + - 'aws-lambda-java-log4j2/**' + - '.github/workflows/aws-lambda-java-log4j2.yml' jobs: build: diff --git a/.github/workflows/aws-lambda-java-serialization.yml b/.github/workflows/aws-lambda-java-serialization.yml index 9d71cbab..83d546ef 100644 --- a/.github/workflows/aws-lambda-java-serialization.yml +++ b/.github/workflows/aws-lambda-java-serialization.yml @@ -7,11 +7,12 @@ on: push: branches: [ main ] paths: - - 'aws-lambda-java-serialization/**' + - 'aws-lambda-java-serialization/**' pull_request: branches: [ '*' ] paths: - - 'aws-lambda-java-serialization/**' + - 'aws-lambda-java-serialization/**' + - '.github/workflows/aws-lambda-java-serialization.yml' jobs: build: diff --git a/.github/workflows/aws-lambda-java-tests.yml b/.github/workflows/aws-lambda-java-tests.yml index fc587e8e..b4f3af00 100644 --- a/.github/workflows/aws-lambda-java-tests.yml +++ b/.github/workflows/aws-lambda-java-tests.yml @@ -7,11 +7,12 @@ on: push: branches: [ main ] paths: - - 'aws-lambda-java-tests/**' + - 'aws-lambda-java-tests/**' pull_request: branches: [ '*' ] paths: - - 'aws-lambda-java-tests/**' + - 'aws-lambda-java-tests/**' + - '.github/workflows/aws-lambda-java-tests.yml' jobs: build: diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 91054f9e..c667bceb 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -3,6 +3,10 @@ name: Repo Sync on: schedule: - cron: "0 8 * * 1-5" # At 08:00 on every day-of-week from Monday through Friday + pull_request: + branches: [ '*' ] + paths: + - '.github/workflows/repo-sync.yml' workflow_dispatch: jobs: diff --git a/.github/workflows/runtime-interface-client_pr.yml b/.github/workflows/runtime-interface-client_pr.yml index 7a7f602e..6aad4a1d 100644 --- a/.github/workflows/runtime-interface-client_pr.yml +++ b/.github/workflows/runtime-interface-client_pr.yml @@ -7,7 +7,8 @@ on: pull_request: branches: [ '*' ] paths: - - 'aws-lambda-java-runtime-interface-client/**' + - 'aws-lambda-java-runtime-interface-client/**' + - '.github/workflows/runtime-interface-client_pr.yml' jobs: diff --git a/.github/workflows/samples.yml b/.github/workflows/samples.yml index 2171ae78..bee5d317 100644 --- a/.github/workflows/samples.yml +++ b/.github/workflows/samples.yml @@ -12,6 +12,7 @@ on: branches: [ '*' ] paths: - 'samples/kinesis-firehose-event-handler/**' + - '.github/workflows/samples.yml' jobs: build: From 70b788f585f1cf7b5aa5373bb32d5df3cee76f3b Mon Sep 17 00:00:00 2001 From: Maxime David Date: Tue, 18 Jun 2024 11:08:31 +0100 Subject: [PATCH 02/14] feat: clean DNS cache via JNI (#51) * feat: clean DNS cache via JNI --- .../workflows/runtime-interface-client_pr.yml | 4 + .../Makefile | 19 ++- .../pom.xml | 26 ++++ .../services/lambda/crac/ContextImpl.java | 43 ++++--- .../services/lambda/crac/DNSManager.java | 10 ++ .../api/client/runtimeapi/JniHelper.java | 64 +++++++++ .../api/client/runtimeapi/NativeClient.java | 62 +-------- .../src/main/jni/Dockerfile.glibc | 9 +- .../src/main/jni/Dockerfile.musl | 9 +- .../src/main/jni/build-jni-lib.sh | 13 +- ...zonaws_services_lambda_crac_DNSManager.cpp | 27 ++++ ...mazonaws_services_lambda_crac_DNSManager.h | 19 +++ ...ime_api_client_runtimeapi_NativeClient.cpp | 22 +--- ...ntime_api_client_runtimeapi_NativeClient.h | 4 + .../src/main/jni/macro.h | 14 ++ .../services/lambda/crac/ContextImplTest.java | 18 ++- .../lambda/crac/DNSCacheManagerTest.java | 121 ++++++++++++++++++ .../codebuild/buildspec.os.alpine.yml | 2 +- .../codebuild/buildspec.os.amazoncorretto.yml | 2 +- .../codebuild/buildspec.os.amazonlinux.1.yml | 2 +- .../codebuild/buildspec.os.amazonlinux.2.yml | 2 +- 21 files changed, 379 insertions(+), 113 deletions(-) create mode 100644 aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/DNSManager.java create mode 100644 aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/JniHelper.java create mode 100644 aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_crac_DNSManager.cpp create mode 100644 aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_crac_DNSManager.h create mode 100644 aws-lambda-java-runtime-interface-client/src/main/jni/macro.h create mode 100644 aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/crac/DNSCacheManagerTest.java diff --git a/.github/workflows/runtime-interface-client_pr.yml b/.github/workflows/runtime-interface-client_pr.yml index 6aad4a1d..081d9d68 100644 --- a/.github/workflows/runtime-interface-client_pr.yml +++ b/.github/workflows/runtime-interface-client_pr.yml @@ -26,6 +26,8 @@ jobs: - name: Runtime Interface Client smoke tests - Run 'pr' target working-directory: ./aws-lambda-java-runtime-interface-client run: make pr + env: + IS_JAVA_8: true build: runs-on: ubuntu-latest @@ -52,6 +54,8 @@ jobs: - name: Test Runtime Interface Client xplatform build - Run 'build' target working-directory: ./aws-lambda-java-runtime-interface-client run: make build + env: + IS_JAVA_8: true - name: Save the built jar uses: actions/upload-artifact@v3 diff --git a/aws-lambda-java-runtime-interface-client/Makefile b/aws-lambda-java-runtime-interface-client/Makefile index e57daf3a..b3a20421 100644 --- a/aws-lambda-java-runtime-interface-client/Makefile +++ b/aws-lambda-java-runtime-interface-client/Makefile @@ -4,6 +4,13 @@ ARCHITECTURE := $(shell arch) ARCHITECTURE_ALIAS := $($(shell echo "$(ARCHITECTURE)_ALIAS")) ARCHITECTURE_ALIAS := $(or $(ARCHITECTURE_ALIAS),amd64) # on any other archs defaulting to amd64 +# Java 8 does not support passing some args (such add --add-opens) so we need to clear them +ifeq ($(IS_JAVA_8),true) + EXTRA_LOAD_ARG := -DargLineForReflectionTestOnly="" +else + EXTRA_LOAD_ARG := +endif + # This optional module exports MAVEN_REPO_URL, MAVEN_REPO_USERNAME and MAVEN_REPO_PASSWORD environment variables # making it possible to publish resulting artifacts to a codeartifact maven repository -include ric-dev-environment/codeartifact-repo.mk @@ -15,7 +22,7 @@ target: .PHONY: test test: - mvn test + mvn test $(EXTRA_LOAD_ARG) .PHONY: setup-codebuild-agent setup-codebuild-agent: @@ -44,11 +51,11 @@ pr: test test-smoke .PHONY: build build: - mvn clean install - mvn install -P linux-x86_64 - mvn install -P linux_musl-x86_64 - mvn install -P linux-aarch64 - mvn install -P linux_musl-aarch64 + mvn clean install $(EXTRA_LOAD_ARG) + mvn install -P linux-x86_64 $(EXTRA_LOAD_ARG) + mvn install -P linux_musl-x86_64 $(EXTRA_LOAD_ARG) + mvn install -P linux-aarch64 $(EXTRA_LOAD_ARG) + mvn install -P linux_musl-aarch64 $(EXTRA_LOAD_ARG) .PHONY: publish publish: diff --git a/aws-lambda-java-runtime-interface-client/pom.xml b/aws-lambda-java-runtime-interface-client/pom.xml index 29d20c8d..2f6a6312 100644 --- a/aws-lambda-java-runtime-interface-client/pom.xml +++ b/aws-lambda-java-runtime-interface-client/pom.xml @@ -47,6 +47,11 @@ + + --add-opens java.base/java.net=ALL-UNNAMED @@ -93,6 +98,9 @@ maven-surefire-plugin 3.0.0-M9 + + ${argLineForReflectionTestOnly} + org.junit.jupiter @@ -110,6 +118,24 @@ maven-antrun-plugin 1.7 + + build-jni-lib-for-tests + generate-test-sources + + run + + + + + + + + + + + + build-jni-lib prepare-package diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/ContextImpl.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/ContextImpl.java index ac4bc648..87fb0ff2 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/ContextImpl.java +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/ContextImpl.java @@ -23,25 +23,9 @@ public class ContextImpl extends Context { @Override public synchronized void beforeCheckpoint(Context context) throws CheckpointException { - - List exceptionsThrown = new ArrayList<>(); - for (Resource resource : getCheckpointQueueReverseOrderOfRegistration()) { - try { - resource.beforeCheckpoint(this); - } catch (CheckpointException e) { - Collections.addAll(exceptionsThrown, e.getSuppressed()); - } catch (Exception e) { - exceptionsThrown.add(e); - } - } - - if (!exceptionsThrown.isEmpty()) { - CheckpointException checkpointException = new CheckpointException(); - for (Throwable t : exceptionsThrown) { - checkpointException.addSuppressed(t); - } - throw checkpointException; - } + executeBeforeCheckpointHooks(); + DNSManager.clearCache(); + System.gc(); } @Override @@ -87,4 +71,25 @@ private List getCheckpointQueueForwardOrderOfRegistration() { .map(Map.Entry::getKey) .collect(Collectors.toList()); } + + private void executeBeforeCheckpointHooks() throws CheckpointException { + List exceptionsThrown = new ArrayList<>(); + for (Resource resource : getCheckpointQueueReverseOrderOfRegistration()) { + try { + resource.beforeCheckpoint(this); + } catch (CheckpointException e) { + Collections.addAll(exceptionsThrown, e.getSuppressed()); + } catch (Exception e) { + exceptionsThrown.add(e); + } + } + + if (!exceptionsThrown.isEmpty()) { + CheckpointException checkpointException = new CheckpointException(); + for (Throwable t : exceptionsThrown) { + checkpointException.addSuppressed(t); + } + throw checkpointException; + } + } } diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/DNSManager.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/DNSManager.java new file mode 100644 index 00000000..4c1a6cd5 --- /dev/null +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/DNSManager.java @@ -0,0 +1,10 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package com.amazonaws.services.lambda.crac; + +class DNSManager { + static native void clearCache(); +} \ No newline at end of file diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/JniHelper.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/JniHelper.java new file mode 100644 index 00000000..82586d27 --- /dev/null +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/JniHelper.java @@ -0,0 +1,64 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +package com.amazonaws.services.lambda.runtime.api.client.runtimeapi; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +public class JniHelper { + + private static final String NATIVE_LIB_PATH = "/tmp/.libaws-lambda-jni.so"; + private static final String NATIVE_CLIENT_JNI_PROPERTY = "com.amazonaws.services.lambda.runtime.api.client.runtimeapi.NativeClient.JNI"; + + /** + * Unpacks JNI library from the JAR to a temporary location and tries to load it using System.load() + * Implementation based on AWS CRT + * (ref. ...) + * + * @param libsToTry - array of native libraries to try + */ + public static void load() { + String jniLib = System.getProperty(NATIVE_CLIENT_JNI_PROPERTY); + if (jniLib != null) { + System.load(jniLib); + } else { + String[] libsToTry = new String[]{ + "libaws-lambda-jni.linux-x86_64.so", + "libaws-lambda-jni.linux-aarch_64.so", + "libaws-lambda-jni.linux_musl-x86_64.so", + "libaws-lambda-jni.linux_musl-aarch_64.so" + }; + unpackAndLoad(libsToTry, NativeClient.class); + } + } + + private static void unpackAndLoad(String[] libsToTry, Class clazz) { + List errorMessages = new ArrayList<>(); + for (String libToTry : libsToTry) { + try (InputStream inputStream = clazz.getResourceAsStream( + Paths.get("/jni", libToTry).toString())) { + if (inputStream == null) { + throw new FileNotFoundException("Specified file not in the JAR: " + libToTry); + } + Files.copy(inputStream, Paths.get(NATIVE_LIB_PATH), StandardCopyOption.REPLACE_EXISTING); + System.load(NATIVE_LIB_PATH); + return; + } catch (UnsatisfiedLinkError | Exception e) { + errorMessages.add(e.getMessage()); + } + } + + for (int i = 0; i < libsToTry.length; ++i) { + System.err.println("Failed to load the native runtime interface client library " + libsToTry[i] + + ". Exception: " + errorMessages.get(i)); + } + System.exit(-1); + } +} diff --git a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/NativeClient.java b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/NativeClient.java index a311ff00..10e6bafd 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/NativeClient.java +++ b/aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/NativeClient.java @@ -5,14 +5,7 @@ package com.amazonaws.services.lambda.runtime.api.client.runtimeapi; import com.amazonaws.services.lambda.runtime.api.client.runtimeapi.dto.InvocationRequest; - -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.List; +import com.amazonaws.services.lambda.runtime.api.client.runtimeapi.JniHelper; import static com.amazonaws.services.lambda.runtime.api.client.runtimeapi.LambdaRuntimeApiClientImpl.USER_AGENT; @@ -21,60 +14,11 @@ * interactions with the Runtime API. */ class NativeClient { - private static final String NATIVE_LIB_PATH = "/tmp/.libaws-lambda-jni.so"; - public static final String NATIVE_CLIENT_JNI_PROPERTY = "com.amazonaws.services.lambda.runtime.api.client.runtimeapi.NativeClient.JNI"; static void init() { - loadJNILib(); + JniHelper.load(); initializeClient(USER_AGENT.getBytes()); } - - private static void loadJNILib() { - String jniLib = System.getProperty(NATIVE_CLIENT_JNI_PROPERTY); - if (jniLib != null) { - System.load(jniLib); - } else { - String[] libsToTry = new String[]{ - "libaws-lambda-jni.linux-x86_64.so", - "libaws-lambda-jni.linux-aarch_64.so", - "libaws-lambda-jni.linux_musl-x86_64.so", - "libaws-lambda-jni.linux_musl-aarch_64.so" - }; - unpackAndLoadNativeLibrary(libsToTry); - } - } - - - /** - * Unpacks JNI library from the JAR to a temporary location and tries to load it using System.load() - * Implementation based on AWS CRT - * (ref. ...) - * - * @param libsToTry - array of native libraries to try - */ - static void unpackAndLoadNativeLibrary(String[] libsToTry) { - - List errorMessages = new ArrayList<>(); - for (String libToTry : libsToTry) { - try (InputStream inputStream = NativeClient.class.getResourceAsStream( - Paths.get("/jni", libToTry).toString())) { - if (inputStream == null) { - throw new FileNotFoundException("Specified file not in the JAR: " + libToTry); - } - Files.copy(inputStream, Paths.get(NATIVE_LIB_PATH), StandardCopyOption.REPLACE_EXISTING); - System.load(NATIVE_LIB_PATH); - return; - } catch (UnsatisfiedLinkError | Exception e) { - errorMessages.add(e.getMessage()); - } - } - - for (int i = 0; i < libsToTry.length; ++i) { - System.err.println("Failed to load the native runtime interface client library " + libsToTry[i] + - ". Exception: " + errorMessages.get(i)); - } - System.exit(-1); - } - + static native void initializeClient(byte[] userAgent); static native InvocationRequest next(); diff --git a/aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.glibc b/aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.glibc index dd4fdb22..1cfcfbb1 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.glibc +++ b/aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.glibc @@ -53,9 +53,16 @@ RUN /usr/bin/c++ -c \ -I${JAVA_HOME}/include/linux \ -I ./deps/artifacts/include \ com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.cpp -o com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o && \ + /usr/bin/c++ -c \ + -std=gnu++11 \ + -fPIC \ + -I${JAVA_HOME}/include \ + -I${JAVA_HOME}/include/linux \ + -I ./deps/artifacts/include \ + com_amazonaws_services_lambda_crac_DNSManager.cpp -o com_amazonaws_services_lambda_crac_DNSManager.o && \ /usr/bin/c++ -shared \ -std=gnu++11 \ - -o aws-lambda-runtime-interface-client.so com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o \ + -o aws-lambda-runtime-interface-client.so com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o com_amazonaws_services_lambda_crac_DNSManager.o \ -L ./deps/artifacts/lib64/ \ -L ./deps/artifacts/lib/ \ -laws-lambda-runtime \ diff --git a/aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.musl b/aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.musl index e15f6adc..64725c14 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.musl +++ b/aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.musl @@ -54,9 +54,16 @@ RUN /usr/bin/c++ -c \ -I${JAVA_HOME}/include/linux \ -I ./deps/artifacts/include \ com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.cpp -o com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o && \ + /usr/bin/c++ -c \ + -std=gnu++11 \ + -fPIC \ + -I${JAVA_HOME}/include \ + -I${JAVA_HOME}/include/linux \ + -I ./deps/artifacts/include \ + com_amazonaws_services_lambda_crac_DNSManager.cpp -o com_amazonaws_services_lambda_crac_DNSManager.o && \ /usr/bin/c++ -shared \ -std=gnu++11 \ - -o aws-lambda-runtime-interface-client.so com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o \ + -o aws-lambda-runtime-interface-client.so com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o com_amazonaws_services_lambda_crac_DNSManager.o \ -L ./deps/artifacts/lib/ \ -laws-lambda-runtime \ -lcurl \ diff --git a/aws-lambda-java-runtime-interface-client/src/main/jni/build-jni-lib.sh b/aws-lambda-java-runtime-interface-client/src/main/jni/build-jni-lib.sh index 3b505e74..b7dbb5a8 100755 --- a/aws-lambda-java-runtime-interface-client/src/main/jni/build-jni-lib.sh +++ b/aws-lambda-java-runtime-interface-client/src/main/jni/build-jni-lib.sh @@ -51,10 +51,21 @@ function build_for_libc_arch() { echo "multi-arch not requested, assuming this is a workaround to goofyness when docker buildx is enabled on Linux CI environments." echo "enabling docker buildx often updates the docker api version, so assuming that docker cli is also too old to use --output type=tar, so doing alternative build-tag-run approach" image_name="lambda-java-jni-lib-${libc_impl}-${arch}" + + # GitHub actions is using dockerx build under the hood. We need to pass --load option to be able to run the image + # This args is NOT part of the classic docker build command, so we need to check against a GitHub Action env var not to make local build crash. + if [[ "${GITHUB_RUN_ID:+isset}" == "isset" ]]; then + EXTRA_LOAD_ARG="--load" + else + EXTRA_LOAD_ARG="" + fi + docker build --platform="${docker_platform}" \ -t "${image_name}" \ -f "${SRC_DIR}/Dockerfile.${libc_impl}" \ - --build-arg CURL_VERSION=${CURL_VERSION} "${SRC_DIR}" + --build-arg CURL_VERSION=${CURL_VERSION} "${SRC_DIR}" ${EXTRA_LOAD_ARG} + + echo "Docker image has been successfully built" docker run --rm --entrypoint /bin/cat "${image_name}" \ /src/aws-lambda-runtime-interface-client.so > "${artifact}" diff --git a/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_crac_DNSManager.cpp b/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_crac_DNSManager.cpp new file mode 100644 index 00000000..ccf5481b --- /dev/null +++ b/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_crac_DNSManager.cpp @@ -0,0 +1,27 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +#include +#include "macro.h" +#include "com_amazonaws_services_lambda_crac_DNSManager.h" + +JNIEXPORT void JNICALL Java_com_amazonaws_services_lambda_crac_DNSManager_clearCache + (JNIEnv *env, jclass thisClass) { + jclass iNetAddressClass; + jclass concurrentMap; + jfieldID cacheFieldID; + jobject cacheObj; + jmethodID clearMethodID; + CHECK_EXCEPTION(env, iNetAddressClass = env->FindClass("java/net/InetAddress")); + CHECK_EXCEPTION(env, concurrentMap = env->FindClass("java/util/concurrent/ConcurrentMap")); + CHECK_EXCEPTION(env, cacheFieldID = env->GetStaticFieldID(iNetAddressClass, "cache", "Ljava/util/concurrent/ConcurrentMap;")); + CHECK_EXCEPTION(env, cacheObj = (jobject) env->GetStaticObjectField(iNetAddressClass, cacheFieldID)); + CHECK_EXCEPTION(env, clearMethodID = env->GetMethodID(concurrentMap, "clear", "()V")); + CHECK_EXCEPTION(env, env->CallVoidMethod(cacheObj, clearMethodID)); + return; + + ERROR: + // we need to fail silently here + env->ExceptionClear(); +} diff --git a/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_crac_DNSManager.h b/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_crac_DNSManager.h new file mode 100644 index 00000000..f26639ba --- /dev/null +++ b/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_crac_DNSManager.h @@ -0,0 +1,19 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +#include + +#ifndef _Included_com_amazonaws_services_lambda_crac_DNSManager +#define _Included_com_amazonaws_services_lambda_crac_DNSManager +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT void JNICALL Java_com_amazonaws_services_lambda_crac_DNSManager_clearCache + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.cpp b/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.cpp index 02cfeb7e..db71b819 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.cpp +++ b/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.cpp @@ -1,27 +1,13 @@ /* - * Copyright 2019-present Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ #include +#include "macro.h" #include "com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.h" #include "aws/lambda-runtime/runtime.h" #include "aws/lambda-runtime/version.h" -#define CHECK_EXCEPTION(env, expr) \ - expr; \ - if ((env)->ExceptionOccurred()) \ - goto ERROR; - static aws::lambda_runtime::runtime * CLIENT = nullptr; static jint JNI_VERSION = JNI_VERSION_1_8; diff --git a/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.h b/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.h index 28a6f444..27c63611 100644 --- a/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.h +++ b/aws-lambda-java-runtime-interface-client/src/main/jni/com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.h @@ -1,3 +1,7 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ #include #ifndef _Included_com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient diff --git a/aws-lambda-java-runtime-interface-client/src/main/jni/macro.h b/aws-lambda-java-runtime-interface-client/src/main/jni/macro.h new file mode 100644 index 00000000..df5759af --- /dev/null +++ b/aws-lambda-java-runtime-interface-client/src/main/jni/macro.h @@ -0,0 +1,14 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +#ifndef _Included_macros +#define _Included_macros + +#define CHECK_EXCEPTION(env, expr) \ + expr; \ + if ((env)->ExceptionOccurred()) \ + goto ERROR; + +#endif diff --git a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/crac/ContextImplTest.java b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/crac/ContextImplTest.java index b8166073..daf956ff 100644 --- a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/crac/ContextImplTest.java +++ b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/crac/ContextImplTest.java @@ -1,22 +1,32 @@ /* - * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. - */ - +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ package com.amazonaws.services.lambda.crac; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; import org.mockito.InOrder; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.doThrow; +import com.amazonaws.services.lambda.runtime.api.client.runtimeapi.JniHelper; + public class ContextImplTest { private Resource throwsWithSuppressedException, noop, noop2, throwsException, throwCustomException; + @BeforeAll + public static void jniLoad() { + JniHelper.load(); + } + @BeforeEach public void setup() throws Exception { diff --git a/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/crac/DNSCacheManagerTest.java b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/crac/DNSCacheManagerTest.java new file mode 100644 index 00000000..7625d15c --- /dev/null +++ b/aws-lambda-java-runtime-interface-client/src/test/java/com/amazonaws/services/lambda/crac/DNSCacheManagerTest.java @@ -0,0 +1,121 @@ +/* +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package com.amazonaws.services.lambda.crac; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.amazonaws.services.lambda.runtime.api.client.runtimeapi.JniHelper; + +import java.util.Map; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class DNSCacheManagerTest { + + static String CACHE_FIELD_NAME = "cache"; + + // this should have no effect, as the DNS cache is cleared explicitly in these tests + static { + java.security.Security.setProperty("networkaddress.cache.ttl" , "10000"); + java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "10000"); + } + + @BeforeAll + public static void jniLoad() { + JniHelper.load(); + } + + @BeforeEach + public void setup() { + Core.resetGlobalContext(); + DNSManager.clearCache(); + } + + static class StatefulResource implements Resource { + + int state = 0; + + @Override + public void afterRestore(Context context) { + state += 1; + } + + @Override + public void beforeCheckpoint(Context context) { + state += 2; + } + + int getValue() { + return state; + } + } + + @Test + public void positiveDnsCacheShouldBeEmpty() throws CheckpointException, RestoreException, UnknownHostException, ReflectiveOperationException { + int baselineDNSEntryCount = getDNSEntryCount(); + + StatefulResource resource = new StatefulResource(); + Core.getGlobalContext().register(resource); + + String[] hosts = {"www.stackoverflow.com", "www.amazon.com", "www.yahoo.com"}; + for(String singleHost : hosts) { + InetAddress address = InetAddress.getByName(singleHost); + } + // n hosts -> n DNS entries + assertEquals(hosts.length, getDNSEntryCount() - baselineDNSEntryCount); + + // this should call the native static method clearDNSCache + Core.getGlobalContext().beforeCheckpoint(null); + + // cache should be cleared + assertEquals(0, getDNSEntryCount()); + } + + @Test + public void negativeDnsCacheShouldBeEmpty() throws CheckpointException, RestoreException, UnknownHostException, ReflectiveOperationException { + int baselineDNSEntryCount = getDNSEntryCount(); + + StatefulResource resource = new StatefulResource(); + Core.getGlobalContext().register(resource); + + String invalidHost = "not.a.valid.host"; + try { + InetAddress address = InetAddress.getByName(invalidHost); + fail(); + } catch(UnknownHostException uhe) { + // this is actually fine + } + + // 1 host -> 1 DNS entry + assertEquals(1, getDNSEntryCount() - baselineDNSEntryCount); + + // this should the native static method clearDNSCache + Core.getGlobalContext().beforeCheckpoint(null); + + // cache should be cleared + assertEquals(0, getDNSEntryCount()); + } + + // helper functions to access the cache via reflection (see maven-surefire-plugin command args) + protected static Map getDNSCache() throws ReflectiveOperationException { + Class klass = InetAddress.class; + Field acf = klass.getDeclaredField(CACHE_FIELD_NAME); + acf.setAccessible(true); + Object addressCache = acf.get(null); + return (Map) acf.get(addressCache); + } + + protected static int getDNSEntryCount() throws ReflectiveOperationException { + Map cache = getDNSCache(); + return cache.size(); + } +} diff --git a/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.alpine.yml b/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.alpine.yml index 4f8caf39..cdc27a65 100644 --- a/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.alpine.yml +++ b/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.alpine.yml @@ -44,7 +44,7 @@ phases: - (cd aws-lambda-java-events && mvn install) # Install serialization (dependency of RIC) - (cd aws-lambda-java-serialization && mvn install) - - (cd aws-lambda-java-runtime-interface-client && mvn install) + - (cd aws-lambda-java-runtime-interface-client && mvn install -DargLineForReflectionTestOnly="") - (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install) - export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}" - echo "Extracting and including Runtime Interface Emulator" diff --git a/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazoncorretto.yml b/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazoncorretto.yml index fd45aabb..67dd7617 100644 --- a/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazoncorretto.yml +++ b/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazoncorretto.yml @@ -43,7 +43,7 @@ phases: - (cd aws-lambda-java-events && mvn install) # Install serialization (dependency of RIC) - (cd aws-lambda-java-serialization && mvn install) - - (cd aws-lambda-java-runtime-interface-client && mvn install) + - (cd aws-lambda-java-runtime-interface-client && mvn install -DargLineForReflectionTestOnly="") - (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install) - export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}" - echo "Extracting and including Runtime Interface Emulator" diff --git a/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.1.yml b/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.1.yml index 197e9724..04c486a8 100644 --- a/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.1.yml +++ b/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.1.yml @@ -38,7 +38,7 @@ phases: - (cd aws-lambda-java-events && mvn install) # Install serialization (dependency of RIC) - (cd aws-lambda-java-serialization && mvn install) - - (cd aws-lambda-java-runtime-interface-client && mvn install -DmultiArch=false) + - (cd aws-lambda-java-runtime-interface-client && mvn install -DmultiArch=false -DargLineForReflectionTestOnly="") - (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install) - export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}" - echo "Extracting and including Runtime Interface Emulator" diff --git a/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.2.yml b/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.2.yml index 75b816e8..8222bb41 100644 --- a/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.2.yml +++ b/aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.2.yml @@ -42,7 +42,7 @@ phases: - (cd aws-lambda-java-events && mvn install) # Install serialization (dependency of RIC) - (cd aws-lambda-java-serialization && mvn install) - - (cd aws-lambda-java-runtime-interface-client && mvn install) + - (cd aws-lambda-java-runtime-interface-client && mvn install -DargLineForReflectionTestOnly="") - (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install) - export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}" - echo "Extracting and including Runtime Interface Emulator" From 379fddd43d6d7983227a258003d130be584619a6 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Tue, 18 Jun 2024 12:44:27 +0100 Subject: [PATCH 03/14] fix: args for maven build / test (#58) --- .github/workflows/runtime-interface-client_merge_to_main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/runtime-interface-client_merge_to_main.yml b/.github/workflows/runtime-interface-client_merge_to_main.yml index 1783f1cd..06dfc4bc 100644 --- a/.github/workflows/runtime-interface-client_merge_to_main.yml +++ b/.github/workflows/runtime-interface-client_merge_to_main.yml @@ -48,6 +48,8 @@ jobs: - name: Test Runtime Interface Client xplatform build - Run 'build' target working-directory: ./aws-lambda-java-runtime-interface-client run: make build + env: + IS_JAVA_8: true - name: Issue AWS credentials uses: aws-actions/configure-aws-credentials@v1 From 3b7dd9648baca067331162979ed2ef8b82e74453 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 19 Jun 2024 20:58:58 +0100 Subject: [PATCH 04/14] fix: snapshot upload (#60) * fix: snapshot upload --- .../runtime-interface-client_merge_to_main.yml | 1 + .gitignore | 5 ++++- aws-lambda-java-runtime-interface-client/pom.xml | 12 ++++++++++++ .../ric-dev-environment/publish_snapshot.sh | 6 +++++- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runtime-interface-client_merge_to_main.yml b/.github/workflows/runtime-interface-client_merge_to_main.yml index 06dfc4bc..646dfb9a 100644 --- a/.github/workflows/runtime-interface-client_merge_to_main.yml +++ b/.github/workflows/runtime-interface-client_merge_to_main.yml @@ -15,6 +15,7 @@ on: branches: [ main ] paths: - 'aws-lambda-java-runtime-interface-client/**' + workflow_dispatch: jobs: diff --git a/.gitignore b/.gitignore index 2f2f0af4..371bed6b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,7 @@ dependency-reduced-pom.xml .project # OSX -.DS_Store \ No newline at end of file +.DS_Store + +# snapshot process +aws-lambda-java-runtime-interface-client/pom.xml.versionsBackup diff --git a/aws-lambda-java-runtime-interface-client/pom.xml b/aws-lambda-java-runtime-interface-client/pom.xml index 2f6a6312..db9d6486 100644 --- a/aws-lambda-java-runtime-interface-client/pom.xml +++ b/aws-lambda-java-runtime-interface-client/pom.xml @@ -35,6 +35,8 @@ UTF-8 UTF-8 0.8.8 + 2.4 + 3.1.1 5.9.2