Skip to content

Commit 70b788f

Browse files
maxdaymaxday
committed
feat: clean DNS cache via JNI (#51)
* feat: clean DNS cache via JNI
1 parent 378cc05 commit 70b788f

21 files changed

+379
-113
lines changed

‎.github/workflows/runtime-interface-client_pr.yml

+4
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ jobs:
2626
- name: Runtime Interface Client smoke tests - Run 'pr' target
2727
working-directory: ./aws-lambda-java-runtime-interface-client
2828
run: make pr
29+
env:
30+
IS_JAVA_8: true
2931

3032
build:
3133
runs-on: ubuntu-latest
@@ -52,6 +54,8 @@ jobs:
5254
- name: Test Runtime Interface Client xplatform build - Run 'build' target
5355
working-directory: ./aws-lambda-java-runtime-interface-client
5456
run: make build
57+
env:
58+
IS_JAVA_8: true
5559

5660
- name: Save the built jar
5761
uses: actions/upload-artifact@v3

‎aws-lambda-java-runtime-interface-client/Makefile

+13-6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@ ARCHITECTURE := $(shell arch)
44
ARCHITECTURE_ALIAS := $($(shell echo "$(ARCHITECTURE)_ALIAS"))
55
ARCHITECTURE_ALIAS := $(or $(ARCHITECTURE_ALIAS),amd64) # on any other archs defaulting to amd64
66

7+
# Java 8 does not support passing some args (such add --add-opens) so we need to clear them
8+
ifeq ($(IS_JAVA_8),true)
9+
EXTRA_LOAD_ARG := -DargLineForReflectionTestOnly=""
10+
else
11+
EXTRA_LOAD_ARG :=
12+
endif
13+
714
# This optional module exports MAVEN_REPO_URL, MAVEN_REPO_USERNAME and MAVEN_REPO_PASSWORD environment variables
815
# making it possible to publish resulting artifacts to a codeartifact maven repository
916
-include ric-dev-environment/codeartifact-repo.mk
@@ -15,7 +22,7 @@ target:
1522

1623
.PHONY: test
1724
test:
18-
mvn test
25+
mvn test $(EXTRA_LOAD_ARG)
1926

2027
.PHONY: setup-codebuild-agent
2128
setup-codebuild-agent:
@@ -44,11 +51,11 @@ pr: test test-smoke
4451

4552
.PHONY: build
4653
build:
47-
mvn clean install
48-
mvn install -P linux-x86_64
49-
mvn install -P linux_musl-x86_64
50-
mvn install -P linux-aarch64
51-
mvn install -P linux_musl-aarch64
54+
mvn clean install $(EXTRA_LOAD_ARG)
55+
mvn install -P linux-x86_64 $(EXTRA_LOAD_ARG)
56+
mvn install -P linux_musl-x86_64 $(EXTRA_LOAD_ARG)
57+
mvn install -P linux-aarch64 $(EXTRA_LOAD_ARG)
58+
mvn install -P linux_musl-aarch64 $(EXTRA_LOAD_ARG)
5259

5360
.PHONY: publish
5461
publish:

‎aws-lambda-java-runtime-interface-client/pom.xml

+26
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@
4747
<target_build_os/>
4848
<target_build_arch/>
4949
<ric.classifier/>
50+
<!--
51+
Some test/integration/codebuild/buildspec.*.yml files will set -DargLineForReflectionTestOnly=""
52+
as this is not supported by all flavors of java
53+
-->
54+
<argLineForReflectionTestOnly>--add-opens java.base/java.net=ALL-UNNAMED</argLineForReflectionTestOnly>
5055
</properties>
5156

5257
<dependencies>
@@ -93,6 +98,9 @@
9398
<plugin>
9499
<artifactId>maven-surefire-plugin</artifactId>
95100
<version>3.0.0-M9</version>
101+
<configuration>
102+
<argLine>${argLineForReflectionTestOnly}</argLine>
103+
</configuration>
96104
<dependencies>
97105
<dependency>
98106
<groupId>org.junit.jupiter</groupId>
@@ -110,6 +118,24 @@
110118
<artifactId>maven-antrun-plugin</artifactId>
111119
<version>1.7</version>
112120
<executions>
121+
<execution>
122+
<id>build-jni-lib-for-tests</id>
123+
<phase>generate-test-sources</phase>
124+
<goals>
125+
<goal>run</goal>
126+
</goals>
127+
<configuration>
128+
<target name="Build JNI libraries for tests">
129+
<exec executable="${project.basedir}/src/main/jni/build-jni-lib.sh"
130+
failonerror="true" logError="true">
131+
<arg value="${project.build.directory}"/>
132+
<arg value="false"/>
133+
<arg value="${target_build_os}"/>
134+
<arg value="${target_build_arch}"/>
135+
</exec>
136+
</target>
137+
</configuration>
138+
</execution>
113139
<execution>
114140
<id>build-jni-lib</id>
115141
<phase>prepare-package</phase>

‎aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/ContextImpl.java

+24-19
Original file line numberDiff line numberDiff line change
@@ -23,25 +23,9 @@ public class ContextImpl extends Context<Resource> {
2323

2424
@Override
2525
public synchronized void beforeCheckpoint(Context<? extends Resource> context) throws CheckpointException {
26-
27-
List<Throwable> exceptionsThrown = new ArrayList<>();
28-
for (Resource resource : getCheckpointQueueReverseOrderOfRegistration()) {
29-
try {
30-
resource.beforeCheckpoint(this);
31-
} catch (CheckpointException e) {
32-
Collections.addAll(exceptionsThrown, e.getSuppressed());
33-
} catch (Exception e) {
34-
exceptionsThrown.add(e);
35-
}
36-
}
37-
38-
if (!exceptionsThrown.isEmpty()) {
39-
CheckpointException checkpointException = new CheckpointException();
40-
for (Throwable t : exceptionsThrown) {
41-
checkpointException.addSuppressed(t);
42-
}
43-
throw checkpointException;
44-
}
26+
executeBeforeCheckpointHooks();
27+
DNSManager.clearCache();
28+
System.gc();
4529
}
4630

4731
@Override
@@ -87,4 +71,25 @@ private List<Resource> getCheckpointQueueForwardOrderOfRegistration() {
8771
.map(Map.Entry::getKey)
8872
.collect(Collectors.toList());
8973
}
74+
75+
private void executeBeforeCheckpointHooks() throws CheckpointException {
76+
List<Throwable> exceptionsThrown = new ArrayList<>();
77+
for (Resource resource : getCheckpointQueueReverseOrderOfRegistration()) {
78+
try {
79+
resource.beforeCheckpoint(this);
80+
} catch (CheckpointException e) {
81+
Collections.addAll(exceptionsThrown, e.getSuppressed());
82+
} catch (Exception e) {
83+
exceptionsThrown.add(e);
84+
}
85+
}
86+
87+
if (!exceptionsThrown.isEmpty()) {
88+
CheckpointException checkpointException = new CheckpointException();
89+
for (Throwable t : exceptionsThrown) {
90+
checkpointException.addSuppressed(t);
91+
}
92+
throw checkpointException;
93+
}
94+
}
9095
}

‎aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/crac/DNSManager.java

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/*
2+
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package com.amazonaws.services.lambda.crac;
7+
8+
class DNSManager {
9+
static native void clearCache();
10+
}

‎aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/JniHelper.java

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
package com.amazonaws.services.lambda.runtime.api.client.runtimeapi;
6+
7+
import java.io.FileNotFoundException;
8+
import java.io.InputStream;
9+
import java.nio.file.Files;
10+
import java.nio.file.Paths;
11+
import java.nio.file.StandardCopyOption;
12+
import java.util.ArrayList;
13+
import java.util.List;
14+
15+
public class JniHelper {
16+
17+
private static final String NATIVE_LIB_PATH = "/tmp/.libaws-lambda-jni.so";
18+
private static final String NATIVE_CLIENT_JNI_PROPERTY = "com.amazonaws.services.lambda.runtime.api.client.runtimeapi.NativeClient.JNI";
19+
20+
/**
21+
* Unpacks JNI library from the JAR to a temporary location and tries to load it using System.load()
22+
* Implementation based on AWS CRT
23+
* (ref. <a href="https://github.com/awslabs/aws-crt-java/blob/0e9c3db8b07258b57c2503cfc47c787ccef10670/src/main/java/software/amazon/awssdk/crt/CRT.java#L106-L134">...</a>)
24+
*
25+
* @param libsToTry - array of native libraries to try
26+
*/
27+
public static void load() {
28+
String jniLib = System.getProperty(NATIVE_CLIENT_JNI_PROPERTY);
29+
if (jniLib != null) {
30+
System.load(jniLib);
31+
} else {
32+
String[] libsToTry = new String[]{
33+
"libaws-lambda-jni.linux-x86_64.so",
34+
"libaws-lambda-jni.linux-aarch_64.so",
35+
"libaws-lambda-jni.linux_musl-x86_64.so",
36+
"libaws-lambda-jni.linux_musl-aarch_64.so"
37+
};
38+
unpackAndLoad(libsToTry, NativeClient.class);
39+
}
40+
}
41+
42+
private static void unpackAndLoad(String[] libsToTry, Class clazz) {
43+
List<String> errorMessages = new ArrayList<>();
44+
for (String libToTry : libsToTry) {
45+
try (InputStream inputStream = clazz.getResourceAsStream(
46+
Paths.get("/jni", libToTry).toString())) {
47+
if (inputStream == null) {
48+
throw new FileNotFoundException("Specified file not in the JAR: " + libToTry);
49+
}
50+
Files.copy(inputStream, Paths.get(NATIVE_LIB_PATH), StandardCopyOption.REPLACE_EXISTING);
51+
System.load(NATIVE_LIB_PATH);
52+
return;
53+
} catch (UnsatisfiedLinkError | Exception e) {
54+
errorMessages.add(e.getMessage());
55+
}
56+
}
57+
58+
for (int i = 0; i < libsToTry.length; ++i) {
59+
System.err.println("Failed to load the native runtime interface client library " + libsToTry[i] +
60+
". Exception: " + errorMessages.get(i));
61+
}
62+
System.exit(-1);
63+
}
64+
}

‎aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/NativeClient.java

+3-59
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,7 @@
55
package com.amazonaws.services.lambda.runtime.api.client.runtimeapi;
66

77
import com.amazonaws.services.lambda.runtime.api.client.runtimeapi.dto.InvocationRequest;
8-
9-
import java.io.FileNotFoundException;
10-
import java.io.InputStream;
11-
import java.nio.file.Files;
12-
import java.nio.file.Paths;
13-
import java.nio.file.StandardCopyOption;
14-
import java.util.ArrayList;
15-
import java.util.List;
8+
import com.amazonaws.services.lambda.runtime.api.client.runtimeapi.JniHelper;
169

1710
import static com.amazonaws.services.lambda.runtime.api.client.runtimeapi.LambdaRuntimeApiClientImpl.USER_AGENT;
1811

@@ -21,60 +14,11 @@
2114
* interactions with the Runtime API.
2215
*/
2316
class NativeClient {
24-
private static final String NATIVE_LIB_PATH = "/tmp/.libaws-lambda-jni.so";
25-
public static final String NATIVE_CLIENT_JNI_PROPERTY = "com.amazonaws.services.lambda.runtime.api.client.runtimeapi.NativeClient.JNI";
2617
static void init() {
27-
loadJNILib();
18+
JniHelper.load();
2819
initializeClient(USER_AGENT.getBytes());
2920
}
30-
31-
private static void loadJNILib() {
32-
String jniLib = System.getProperty(NATIVE_CLIENT_JNI_PROPERTY);
33-
if (jniLib != null) {
34-
System.load(jniLib);
35-
} else {
36-
String[] libsToTry = new String[]{
37-
"libaws-lambda-jni.linux-x86_64.so",
38-
"libaws-lambda-jni.linux-aarch_64.so",
39-
"libaws-lambda-jni.linux_musl-x86_64.so",
40-
"libaws-lambda-jni.linux_musl-aarch_64.so"
41-
};
42-
unpackAndLoadNativeLibrary(libsToTry);
43-
}
44-
}
45-
46-
47-
/**
48-
* Unpacks JNI library from the JAR to a temporary location and tries to load it using System.load()
49-
* Implementation based on AWS CRT
50-
* (ref. <a href="https://github.com/awslabs/aws-crt-java/blob/0e9c3db8b07258b57c2503cfc47c787ccef10670/src/main/java/software/amazon/awssdk/crt/CRT.java#L106-L134">...</a>)
51-
*
52-
* @param libsToTry - array of native libraries to try
53-
*/
54-
static void unpackAndLoadNativeLibrary(String[] libsToTry) {
55-
56-
List<String> errorMessages = new ArrayList<>();
57-
for (String libToTry : libsToTry) {
58-
try (InputStream inputStream = NativeClient.class.getResourceAsStream(
59-
Paths.get("/jni", libToTry).toString())) {
60-
if (inputStream == null) {
61-
throw new FileNotFoundException("Specified file not in the JAR: " + libToTry);
62-
}
63-
Files.copy(inputStream, Paths.get(NATIVE_LIB_PATH), StandardCopyOption.REPLACE_EXISTING);
64-
System.load(NATIVE_LIB_PATH);
65-
return;
66-
} catch (UnsatisfiedLinkError | Exception e) {
67-
errorMessages.add(e.getMessage());
68-
}
69-
}
70-
71-
for (int i = 0; i < libsToTry.length; ++i) {
72-
System.err.println("Failed to load the native runtime interface client library " + libsToTry[i] +
73-
". Exception: " + errorMessages.get(i));
74-
}
75-
System.exit(-1);
76-
}
77-
21+
7822
static native void initializeClient(byte[] userAgent);
7923

8024
static native InvocationRequest next();

‎aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.glibc

+8-1
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,16 @@ RUN /usr/bin/c++ -c \
5353
-I${JAVA_HOME}/include/linux \
5454
-I ./deps/artifacts/include \
5555
com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.cpp -o com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o && \
56+
/usr/bin/c++ -c \
57+
-std=gnu++11 \
58+
-fPIC \
59+
-I${JAVA_HOME}/include \
60+
-I${JAVA_HOME}/include/linux \
61+
-I ./deps/artifacts/include \
62+
com_amazonaws_services_lambda_crac_DNSManager.cpp -o com_amazonaws_services_lambda_crac_DNSManager.o && \
5663
/usr/bin/c++ -shared \
5764
-std=gnu++11 \
58-
-o aws-lambda-runtime-interface-client.so com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o \
65+
-o aws-lambda-runtime-interface-client.so com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o com_amazonaws_services_lambda_crac_DNSManager.o \
5966
-L ./deps/artifacts/lib64/ \
6067
-L ./deps/artifacts/lib/ \
6168
-laws-lambda-runtime \

‎aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.musl

+8-1
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,16 @@ RUN /usr/bin/c++ -c \
5454
-I${JAVA_HOME}/include/linux \
5555
-I ./deps/artifacts/include \
5656
com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.cpp -o com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o && \
57+
/usr/bin/c++ -c \
58+
-std=gnu++11 \
59+
-fPIC \
60+
-I${JAVA_HOME}/include \
61+
-I${JAVA_HOME}/include/linux \
62+
-I ./deps/artifacts/include \
63+
com_amazonaws_services_lambda_crac_DNSManager.cpp -o com_amazonaws_services_lambda_crac_DNSManager.o && \
5764
/usr/bin/c++ -shared \
5865
-std=gnu++11 \
59-
-o aws-lambda-runtime-interface-client.so com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o \
66+
-o aws-lambda-runtime-interface-client.so com_amazonaws_services_lambda_runtime_api_client_runtimeapi_NativeClient.o com_amazonaws_services_lambda_crac_DNSManager.o \
6067
-L ./deps/artifacts/lib/ \
6168
-laws-lambda-runtime \
6269
-lcurl \

‎aws-lambda-java-runtime-interface-client/src/main/jni/build-jni-lib.sh

+12-1
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,21 @@ function build_for_libc_arch() {
5151
echo "multi-arch not requested, assuming this is a workaround to goofyness when docker buildx is enabled on Linux CI environments."
5252
echo "enabling docker buildx often updates the docker api version, so assuming that docker cli is also too old to use --output type=tar, so doing alternative build-tag-run approach"
5353
image_name="lambda-java-jni-lib-${libc_impl}-${arch}"
54+
55+
# GitHub actions is using dockerx build under the hood. We need to pass --load option to be able to run the image
56+
# This args is NOT part of the classic docker build command, so we need to check against a GitHub Action env var not to make local build crash.
57+
if [[ "${GITHUB_RUN_ID:+isset}" == "isset" ]]; then
58+
EXTRA_LOAD_ARG="--load"
59+
else
60+
EXTRA_LOAD_ARG=""
61+
fi
62+
5463
docker build --platform="${docker_platform}" \
5564
-t "${image_name}" \
5665
-f "${SRC_DIR}/Dockerfile.${libc_impl}" \
57-
--build-arg CURL_VERSION=${CURL_VERSION} "${SRC_DIR}"
66+
--build-arg CURL_VERSION=${CURL_VERSION} "${SRC_DIR}" ${EXTRA_LOAD_ARG}
67+
68+
echo "Docker image has been successfully built"
5869

5970
docker run --rm --entrypoint /bin/cat "${image_name}" \
6071
/src/aws-lambda-runtime-interface-client.so > "${artifact}"

0 commit comments

Comments
 (0)