Skip to content

Commit 29a27d6

Browse files
zsombor-baloghzsombor-balogh
authored
Fix os compatibility tests by enabling multi-platform build and testing (#334)
* Fix os compatibility tests by enabling multi-platform build and testing * Extract environment setup script
1 parent 43610a5 commit 29a27d6

20 files changed

+159
-72
lines changed

‎aws-lambda-java-runtime-interface-client/Makefile

+13-3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
x86_64_ALIAS := amd64
2+
aarch64_ALIAS := arm64
3+
ARCHITECTURE := $(shell arch)
4+
ARCHITECTURE_ALIAS := $($(shell echo "$(ARCHITECTURE)_ALIAS"))
5+
ARCHITECTURE_ALIAS := $(or $(ARCHITECTURE_ALIAS),amd64) # on any other archs defaulting to amd64
6+
17
.PHONY: target
28
target:
39
$(info ${HELP_MESSAGE})
@@ -9,12 +15,16 @@ test:
915

1016
.PHONY: setup-codebuild-agent
1117
setup-codebuild-agent:
12-
docker build -t codebuild-agent - < test/integration/codebuild-local/Dockerfile.agent
18+
docker build -t codebuild-agent \
19+
--build-arg ARCHITECTURE=$(ARCHITECTURE_ALIAS) \
20+
- < test/integration/codebuild-local/Dockerfile.agent
1321

1422
.PHONY: test-smoke
1523
test-smoke: setup-codebuild-agent
16-
CODEBUILD_IMAGE_TAG=codebuild-agent test/integration/codebuild-local/test_one.sh test/integration/codebuild/buildspec.os.alpine.yml alpine 3.12 corretto11
17-
CODEBUILD_IMAGE_TAG=codebuild-agent test/integration/codebuild-local/test_one.sh test/integration/codebuild/buildspec.os.amazoncorretto.yml amazoncorretto amazoncorretto 11
24+
CODEBUILD_IMAGE_TAG=codebuild-agent test/integration/codebuild-local/test_one.sh test/integration/codebuild/buildspec.os.alpine.yml alpine 3.15 corretto11 linux/amd64
25+
CODEBUILD_IMAGE_TAG=codebuild-agent test/integration/codebuild-local/test_one.sh test/integration/codebuild/buildspec.os.alpine.yml alpine 3.15 corretto11 linux/arm64/v8
26+
CODEBUILD_IMAGE_TAG=codebuild-agent test/integration/codebuild-local/test_one.sh test/integration/codebuild/buildspec.os.amazoncorretto.yml amazoncorretto amazoncorretto 11 linux/amd64
27+
CODEBUILD_IMAGE_TAG=codebuild-agent test/integration/codebuild-local/test_one.sh test/integration/codebuild/buildspec.os.amazoncorretto.yml amazoncorretto amazoncorretto 11 linux/arm64/v8
1828

1929
.PHONY: test-integ
2030
test-integ: setup-codebuild-agent

‎aws-lambda-java-runtime-interface-client/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ The Runtime Interface Client library can be installed into the image separate fr
2424
Dockerfile
2525
```dockerfile
2626
# we'll use Amazon Linux 2 + Corretto 11 as our base
27-
FROM amazoncorretto:11 as base
27+
FROM public.ecr.aws/amazoncorretto/amazoncorretto:11 as base
2828

2929
# configure the build environment
3030
FROM base as build

‎aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.glibc

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# we use centos 7 to build against glibc 2.17
2-
FROM centos:7
2+
FROM public.ecr.aws/docker/library/centos:7
33

44
ARG CURL_VERSION
55

‎aws-lambda-java-runtime-interface-client/src/main/jni/Dockerfile.musl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM alpine:3
1+
FROM public.ecr.aws/docker/library/alpine:3
22

33
ARG CURL_VERSION
44

‎aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/Dockerfile.agent

+10-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
FROM public.ecr.aws/amazoncorretto/amazoncorretto:8
22

3+
ARG ARCHITECTURE="amd64"
4+
5+
ENV DOCKER_CLI_PLUGIN_DIR="/root/.docker/cli-plugins"
6+
37
RUN amazon-linux-extras enable docker && \
48
yum clean metadata && \
5-
yum install -y docker tar maven unzip file
9+
yum install -y docker tar maven unzip file wget
10+
RUN mkdir -p "${DOCKER_CLI_PLUGIN_DIR}"
11+
RUN wget \
12+
"$(curl https://api.github.com/repos/docker/buildx/releases/latest | grep browser_download_url | grep "linux-${ARCHITECTURE}" | cut -d '"' -f 4)" \
13+
-O "${DOCKER_CLI_PLUGIN_DIR}"/docker-buildx
14+
RUN chmod +x "${DOCKER_CLI_PLUGIN_DIR}"/docker-buildx

‎aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/test_all.sh

+17-12
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,19 @@ do_one_yaml() {
1818

1919
OS_DISTRIBUTION=$(grep -oE 'OS_DISTRIBUTION:\s*(\S+)' "$YML" | cut -d' ' -f2)
2020
DISTRO_VERSIONS=$(sed '1,/DISTRO_VERSION/d;/RUNTIME_VERSION/,$d' "$YML" | tr -d '\-" ')
21-
RUNTIME_VERSIONS=$(sed '1,/RUNTIME_VERSION/d;/phases/,$d' "$YML" | sed '/#.*$/d' | tr -d '\-" ')
21+
RUNTIME_VERSIONS=$(sed '1,/RUNTIME_VERSION/d;/PLATFORM/,$d' "$YML" | sed '/#.*$/d' | tr -d '\-" ')
22+
PLATFORMS=$(sed '1,/PLATFORM/d;/phases/,$d' "$YML" | tr -d '\-" ')
2223

23-
for DISTRO_VERSION in $DISTRO_VERSIONS ; do
24-
for RUNTIME_VERSION in $RUNTIME_VERSIONS ; do
25-
if (( DRYRUN == 1 )) ; then
26-
echo DRYRUN test_one_combination "$YML" "$OS_DISTRIBUTION" "$DISTRO_VERSION" "$RUNTIME_VERSION"
27-
else
28-
test_one_combination "$YML" "$OS_DISTRIBUTION" "$DISTRO_VERSION" "$RUNTIME_VERSION"
29-
fi
24+
for DISTRO_VERSION in $DISTRO_VERSIONS; do
25+
for RUNTIME_VERSION in $RUNTIME_VERSIONS; do
26+
for PLATFORM in $PLATFORMS; do
27+
if (( DRYRUN == 1 )); then
28+
echo DRYRUN test_one_combination "$YML" "$OS_DISTRIBUTION" "$DISTRO_VERSION" "$RUNTIME_VERSION" "$PLATFORM"
29+
else
30+
test_one_combination "$YML" "$OS_DISTRIBUTION" "$DISTRO_VERSION" "$RUNTIME_VERSION" "$PLATFORM"
31+
fi
3032
done
33+
done
3134
done
3235
}
3336

@@ -36,13 +39,15 @@ test_one_combination() {
3639
local -r OS_DISTRIBUTION="$2"
3740
local -r DISTRO_VERSION="$3"
3841
local -r RUNTIME_VERSION="$4"
39-
42+
local -r PLATFORM="$5"
43+
local -r PLATFORM_SANITIZED=$(echo "$PLATFORM" | tr "/" ".")
44+
4045
echo Testing:
4146
echo " BUILDSPEC" "$YML"
42-
echo " with" "$OS_DISTRIBUTION"-"$DISTRO_VERSION" "$RUNTIME_VERSION"
47+
echo " with" "$OS_DISTRIBUTION"-"$DISTRO_VERSION" "$RUNTIME_VERSION" "$PLATFORM"
4348

44-
"$(dirname "$0")"/test_one.sh "$YML" "$OS_DISTRIBUTION" "$DISTRO_VERSION" "$RUNTIME_VERSION" \
45-
> >(sed "s/^/$OS_DISTRIBUTION$DISTRO_VERSION-$RUNTIME_VERSION: /") 2> >(sed "s/^/$OS_DISTRIBUTION-$DISTRO_VERSION:$RUNTIME_VERSION: /" >&2)
49+
"$(dirname "$0")"/test_one.sh "$YML" "$OS_DISTRIBUTION" "$DISTRO_VERSION" "$RUNTIME_VERSION" "$PLATFORM" \
50+
> >(sed "s/^/$OS_DISTRIBUTION$DISTRO_VERSION-$RUNTIME_VERSION-$PLATFORM_SANITIZED: /") 2> >(sed "s/^/$OS_DISTRIBUTION-$DISTRO_VERSION:$RUNTIME_VERSION:$PLATFORM_SANITIZED: /" >&2)
4651
}
4752

4853
main() {

‎aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/test_one.sh

+7-3
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,13 @@ function usage {
1313
>&2 echo " os_distribution Used to specify the OS distribution to build."
1414
>&2 echo " distro_version Used to specify the distro version of <os_distribution>."
1515
>&2 echo " runtime_version Used to specify the runtime version to test on the selected <distro_version>."
16+
>&2 echo " platform Used to specify the architecture platform to test on the selected <distro_version>."
1617
>&2 echo "Optional:"
1718
>&2 echo " env Additional environment variables file."
1819
}
1920

2021
main() {
21-
if (( $# != 3 && $# != 4)); then
22+
if (( $# != 5 && $# != 6)); then
2223
>&2 echo "Invalid number of parameters."
2324
usage
2425
exit 1
@@ -28,9 +29,11 @@ main() {
2829
OS_DISTRIBUTION="$2"
2930
DISTRO_VERSION="$3"
3031
RUNTIME_VERSION="$4"
31-
EXTRA_ENV="${5-}"
32+
PLATFORM="$5"
33+
PLATFORM_SANITIZED=$(echo "$PLATFORM" | tr "/" ".")
34+
EXTRA_ENV="${6-}"
3235

33-
CODEBUILD_TEMP_DIR=$(mktemp -d codebuild."$OS_DISTRIBUTION"-"$DISTRO_VERSION"-"$RUNTIME_VERSION".XXXXXXXXXX)
36+
CODEBUILD_TEMP_DIR=$(mktemp -d codebuild."$OS_DISTRIBUTION"-"$DISTRO_VERSION"-"$RUNTIME_VERSION"-"$PLATFORM_SANITIZED".XXXXXXXXXX)
3437
trap 'rm -rf $CODEBUILD_TEMP_DIR' EXIT
3538

3639
# Create an env file for codebuild_build.
@@ -43,6 +46,7 @@ main() {
4346
echo "OS_DISTRIBUTION=$OS_DISTRIBUTION"
4447
echo "DISTRO_VERSION=$DISTRO_VERSION"
4548
echo "RUNTIME_VERSION=$RUNTIME_VERSION"
49+
echo "PLATFORM=$PLATFORM"
4650
} >> "$ENVFILE"
4751

4852
ARTIFACTS_DIR="$CODEBUILD_TEMP_DIR/artifacts"

‎aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.alpine.yml

+14-9
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ env:
44
variables:
55
OS_DISTRIBUTION: alpine
66
JAVA_BINARY_LOCATION: "/usr/bin/java"
7+
DOCKER_CLI_EXPERIMENTAL: "enabled"
8+
DOCKER_CLI_PLUGIN_DIR: "/root/.docker/cli-plugins"
79
batch:
810
build-matrix:
911
static:
@@ -14,12 +16,14 @@ batch:
1416
env:
1517
variables:
1618
DISTRO_VERSION:
17-
- "3.9"
18-
- "3.10"
19-
- "3.11"
20-
- "3.12"
19+
- "3.13"
20+
- "3.14"
21+
- "3.15"
2122
RUNTIME_VERSION:
2223
- "corretto11"
24+
PLATFORM:
25+
- "linux/amd64"
26+
- "linux/arm64/v8"
2327
phases:
2428
install:
2529
commands:
@@ -31,6 +35,7 @@ phases:
3135
echo "Performing DockerHub login . . ."
3236
docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
3337
fi
38+
- aws-lambda-java-runtime-interface-client/test/integration/codebuild/scripts/configure_multi_arch_env.sh
3439
pre_build:
3540
commands:
3641
# Log some environment variables for troubleshooting
@@ -39,20 +44,19 @@ phases:
3944
- (cd aws-lambda-java-events && mvn install)
4045
# Install serialization (dependency of RIC)
4146
- (cd aws-lambda-java-serialization && mvn install)
42-
- (cd aws-lambda-java-runtime-interface-client && mvn install -DmultiArch=false)
47+
- (cd aws-lambda-java-runtime-interface-client && mvn install)
4348
- (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install)
4449
- export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}"
4550
- echo "Extracting and including Runtime Interface Emulator"
4651
- SCRATCH_DIR=".scratch"
4752
- mkdir "${SCRATCH_DIR}"
48-
- ARCHITECTURE=$(arch)
4953
- >
50-
if [[ "$ARCHITECTURE" == "x86_64" ]]; then
54+
if [[ "$PLATFORM" == "linux/amd64" ]]; then
5155
RIE="aws-lambda-rie"
52-
elif [[ "$ARCHITECTURE" == "aarch64" ]]; then
56+
elif [[ "$PLATFORM" == "linux/arm64/v8" ]]; then
5357
RIE="aws-lambda-rie-arm64"
5458
else
55-
echo "Architecture $ARCHITECTURE is not currently supported."
59+
echo "Platform $PLATFORM is not currently supported."
5660
exit 1
5761
fi
5862
- tar -xvf aws-lambda-java-runtime-interface-client/test/integration/resources/${RIE}.tar.gz --directory "${SCRATCH_DIR}"
@@ -70,6 +74,7 @@ phases:
7074
docker build . \
7175
-f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
7276
-t "${IMAGE_TAG}" \
77+
--platform="${PLATFORM}" \
7378
--build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \
7479
--build-arg DISTRO_VERSION="${DISTRO_VERSION}"
7580
build:

‎aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazoncorretto.yml

+11-5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ env:
44
variables:
55
OS_DISTRIBUTION: amazoncorretto
66
JAVA_BINARY_LOCATION: "/usr/bin/java"
7+
DOCKER_CLI_EXPERIMENTAL: "enabled"
8+
DOCKER_CLI_PLUGIN_DIR: "/root/.docker/cli-plugins"
79
batch:
810
build-matrix:
911
static:
@@ -18,6 +20,9 @@ batch:
1820
RUNTIME_VERSION:
1921
- "8"
2022
- "11"
23+
PLATFORM:
24+
- "linux/amd64"
25+
- "linux/arm64/v8"
2126
phases:
2227
install:
2328
commands:
@@ -29,6 +34,7 @@ phases:
2934
echo "Performing DockerHub login . . ."
3035
docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
3136
fi
37+
- aws-lambda-java-runtime-interface-client/test/integration/codebuild/scripts/configure_multi_arch_env.sh
3238
pre_build:
3339
commands:
3440
# Log some environment variables for troubleshooting
@@ -37,20 +43,19 @@ phases:
3743
- (cd aws-lambda-java-events && mvn install)
3844
# Install serialization (dependency of RIC)
3945
- (cd aws-lambda-java-serialization && mvn install)
40-
- (cd aws-lambda-java-runtime-interface-client && mvn install -DmultiArch=false)
46+
- (cd aws-lambda-java-runtime-interface-client && mvn install)
4147
- (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install)
4248
- export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}"
4349
- echo "Extracting and including Runtime Interface Emulator"
4450
- SCRATCH_DIR=".scratch"
4551
- mkdir "${SCRATCH_DIR}"
46-
- ARCHITECTURE=$(arch)
4752
- >
48-
if [[ "$ARCHITECTURE" == "x86_64" ]]; then
53+
if [[ "$PLATFORM" == "linux/amd64" ]]; then
4954
RIE="aws-lambda-rie"
50-
elif [[ "$ARCHITECTURE" == "aarch64" ]]; then
55+
elif [[ "$PLATFORM" == "linux/arm64/v8" ]]; then
5156
RIE="aws-lambda-rie-arm64"
5257
else
53-
echo "Architecture $ARCHITECTURE is not currently supported."
58+
echo "Platform $PLATFORM is not currently supported."
5459
exit 1
5560
fi
5661
- tar -xvf aws-lambda-java-runtime-interface-client/test/integration/resources/${RIE}.tar.gz --directory "${SCRATCH_DIR}"
@@ -65,6 +70,7 @@ phases:
6570
docker build . \
6671
-f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
6772
-t "${IMAGE_TAG}" \
73+
--platform="${PLATFORM}" \
6874
--build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \
6975
--build-arg DISTRO_VERSION="${DISTRO_VERSION}"
7076
build:

‎aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.1.yml

+4-11
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ batch:
1717
- "1"
1818
RUNTIME_VERSION:
1919
- "openjdk8"
20+
PLATFORM:
21+
- "linux/amd64"
2022
phases:
2123
install:
2224
commands:
@@ -36,22 +38,13 @@ phases:
3638
- (cd aws-lambda-java-events && mvn install)
3739
# Install serialization (dependency of RIC)
3840
- (cd aws-lambda-java-serialization && mvn install)
39-
- (cd aws-lambda-java-runtime-interface-client && mvn install)
41+
- (cd aws-lambda-java-runtime-interface-client && mvn install -DmultiArch=false)
4042
- (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install)
4143
- export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}"
4244
- echo "Extracting and including Runtime Interface Emulator"
4345
- SCRATCH_DIR=".scratch"
4446
- mkdir "${SCRATCH_DIR}"
45-
- ARCHITECTURE=$(arch)
46-
- >
47-
if [[ "$ARCHITECTURE" == "x86_64" ]]; then
48-
RIE="aws-lambda-rie"
49-
elif [[ "$ARCHITECTURE" == "aarch64" ]]; then
50-
RIE="aws-lambda-rie-arm64"
51-
else
52-
echo "Architecture $ARCHITECTURE is not currently supported."
53-
exit 1
54-
fi
47+
- RIE="aws-lambda-rie"
5548
- tar -xvf aws-lambda-java-runtime-interface-client/test/integration/resources/${RIE}.tar.gz --directory "${SCRATCH_DIR}"
5649
- >
5750
cp "aws-lambda-java-runtime-interface-client/test/integration/docker/Dockerfile.function.${OS_DISTRIBUTION}" \

‎aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazonlinux.2.yml

+10-4
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ env:
44
variables:
55
OS_DISTRIBUTION: amazonlinux
66
JAVA_BINARY_LOCATION: "/usr/bin/java"
7+
DOCKER_CLI_EXPERIMENTAL: "enabled"
8+
DOCKER_CLI_PLUGIN_DIR: "/root/.docker/cli-plugins"
79
batch:
810
build-matrix:
911
static:
@@ -17,6 +19,9 @@ batch:
1719
- "2"
1820
RUNTIME_VERSION:
1921
- "openjdk8"
22+
PLATFORM:
23+
- "linux/amd64"
24+
- "linux/arm64/v8"
2025
phases:
2126
install:
2227
commands:
@@ -28,6 +33,7 @@ phases:
2833
echo "Performing DockerHub login . . ."
2934
docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
3035
fi
36+
- aws-lambda-java-runtime-interface-client/test/integration/codebuild/scripts/configure_multi_arch_env.sh
3137
pre_build:
3238
commands:
3339
# Log some environment variables for troubleshooting
@@ -42,14 +48,13 @@ phases:
4248
- echo "Extracting and including Runtime Interface Emulator"
4349
- SCRATCH_DIR=".scratch"
4450
- mkdir "${SCRATCH_DIR}"
45-
- ARCHITECTURE=$(arch)
4651
- >
47-
if [[ "$ARCHITECTURE" == "x86_64" ]]; then
52+
if [[ "$PLATFORM" == "linux/amd64" ]]; then
4853
RIE="aws-lambda-rie"
49-
elif [[ "$ARCHITECTURE" == "aarch64" ]]; then
54+
elif [[ "$PLATFORM" == "linux/arm64/v8" ]]; then
5055
RIE="aws-lambda-rie-arm64"
5156
else
52-
echo "Architecture $ARCHITECTURE is not currently supported."
57+
echo "Platform $PLATFORM is not currently supported."
5358
exit 1
5459
fi
5560
- tar -xvf aws-lambda-java-runtime-interface-client/test/integration/resources/${RIE}.tar.gz --directory "${SCRATCH_DIR}"
@@ -64,6 +69,7 @@ phases:
6469
docker build . \
6570
-f "${SCRATCH_DIR}/Dockerfile.function.${OS_DISTRIBUTION}.tmp" \
6671
-t "${IMAGE_TAG}" \
72+
--platform="${PLATFORM}" \
6773
--build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \
6874
--build-arg DISTRO_VERSION="${DISTRO_VERSION}"
6975
build:

0 commit comments

Comments
 (0)