Ensure at build time that the jni parts of the runtime-interface-client are compiled for the correct architectures

Author: bmoffatt

.github/workflows/aws-lambda-java-runtime-interface-client.yml

@@ -8,25 +8,56 @@ on:
     branches: [ master ]
     paths:
     - 'aws-lambda-java-runtime-interface-client/**'
+    - '.github/workflows/**'
   pull_request:
     branches: [ '*' ]
     paths:
     - 'aws-lambda-java-runtime-interface-client/**'
 
 jobs:
-  build:
 
+  smoke-test:
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up JDK 1.8
       uses: actions/setup-java@v1
       with:
         java-version: 1.8
-        
-    # Test Runtime Interface Client
-    - name: Run 'pr' target
+
+    - name: Runtime Interface Client smoke tests - Run 'pr' target
       working-directory: ./aws-lambda-java-runtime-interface-client
       run: make pr
 
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        install: true
+
+    - name: Available buildx platforms
+      run: echo ${{ steps.buildx.outputs.platforms }}
+          
+    - name: Test Runtime Interface Client xplatform build - Run 'build' target
+      working-directory: ./aws-lambda-java-runtime-interface-client
+      run: make build
+      
+    - name: Save the built jar
+      uses: actions/upload-artifact@v3
+      with:
+        name: the-jar
+        path: ./aws-lambda-java-runtime-interface-client/target/aws-lambda-java-runtime-interface-client-2.1.0.jar
+

aws-lambda-java-runtime-interface-client/pom.xml

@@ -30,12 +30,21 @@
     </developer>
   </developers>
 
+
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
     <jacoco.maven.plugin.version>0.8.7</jacoco.maven.plugin.version>
+    <!-- 
+        The test/integration/codebuild/buildspec.*.yml files will set -DmultiArch=false
+        as a workaround for executing within Github Actions. At time of writing (2022-04-08) the 
+        integration tests, `make pr,` do not play nice with mixed `docker build`, `docker run`, 
+        when docker buildx+qemu is configured on Linux. So we'll test the multi-platform artifact build 
+        separately from the Runtime Interface Client functionality until we figure something else out.
+    -->
+    <multiArch>true</multiArch>
   </properties>
 
   <dependencies>
@@ -96,6 +105,7 @@
               <target name="Build JNI libraries">
                 <exec executable="${project.basedir}/src/main/jni/build-jni-lib.sh" failonerror="true" logError="true">
                   <arg value="${project.build.directory}"/>
+                  <arg value="${multiArch}"/>
                 </exec>
               </target>
             </configuration>

aws-lambda-java-runtime-interface-client/src/main/java/com/amazonaws/services/lambda/runtime/api/client/runtimeapi/NativeClient.java

@@ -59,7 +59,7 @@ static String getArchIdentifier() {
         if (arch.matches(supported_x86_architectures)) {
             return "x86_64";
         } else if (arch.matches(supported_arm_architectures)) {
-            return "arm64";
+            return "aarch64";
         }
 
         throw new UnknownPlatformException("architecture not supported: " + arch);

aws-lambda-java-runtime-interface-client/src/main/jni/build-jni-lib.sh

@@ -5,14 +5,15 @@ set -euo pipefail
 
 SRC_DIR=$(dirname "$0")
 DST_DIR=${1}
+MULTI_ARCH=${2}
 CURL_VERSION=7.77.0
 
 # Not using associative arrays to maintain bash 3 compatibility with building on MacOS
 # MacOS ships with bash 3 and associative arrays require bash 4+
 # Declaring a map as an array with the column character as a separator :
 declare -a ARCHITECTURES_TO_PLATFORM=(
     "x86_64:linux/amd64"
-    "arm64:linux/arm64/v8"
+    "aarch64:linux/arm64/v8"
 )
 
 declare -a TARGETS=("glibc" "musl")
@@ -21,11 +22,30 @@ for pair in "${ARCHITECTURES_TO_PLATFORM[@]}"; do
     arch=${pair%%:*}
     platform=${pair#*:}
 
-    mkdir -p "${DST_DIR}"/classes/"${arch}"
+    if [[ "${MULTI_ARCH}" != "true" ]] && [[ "$(arch)" != "${arch}" ]]; then
+        echo "multi arch build not requested and host arch is $(arch), so skipping ${arch}:${platform} ..."
+        continue
+    fi
+
+    mkdir -p "${DST_DIR}/classes/${arch}"
 
     for target in "${TARGETS[@]}"; do
         echo "Compiling the native library for target ${target} on architecture ${arch} using Docker platform ${platform}"
-        docker build --platform="${platform}" -f "${SRC_DIR}"/Dockerfile."${target}" --build-arg CURL_VERSION=${CURL_VERSION} -t lambda-java-jni-lib-"${target}"-"${arch}" "${SRC_DIR}"
-        docker run --rm --entrypoint /bin/cat lambda-java-jni-lib-"${target}"-"${arch}" /src/aws-lambda-runtime-interface-client.so > "${DST_DIR}"/classes/"${arch}"/aws-lambda-runtime-interface-client."${target}".so
+        artifact="${DST_DIR}/classes/${arch}/aws-lambda-runtime-interface-client.${target}.so"
+
+        if [[ "${MULTI_ARCH}" == "true" ]]; then
+            docker build --platform="${platform}" -f "${SRC_DIR}/Dockerfile.${target}" --build-arg CURL_VERSION=${CURL_VERSION} "${SRC_DIR}" -o - | tar -xOf - src/aws-lambda-runtime-interface-client.so > "${artifact}"
+        else
+            echo "multi-arch not requestsed, assuming this is a workaround to goofyness when docker buildx is enabled on Linux CI environments."
+            echo "enabling docker buildx often updates the docker api version, so assuming that docker cli is also too old to use --output type=tar, so doing alternative build-tag-run approach"
+            docker build --platform="${platform}" -t "lambda-java-jni-lib-${target}-${arch}" -f "${SRC_DIR}/Dockerfile.${target}" --build-arg CURL_VERSION=${CURL_VERSION} "${SRC_DIR}"
+            docker run --rm --entrypoint /bin/cat "lambda-java-jni-lib-${target}-${arch}" /src/aws-lambda-runtime-interface-client.so > "${artifact}"
+        fi
+
+        [ -f "${artifact}" ]
+        if ! file -b "${artifact}" | tr '-' '_' | tee /dev/stderr | grep -q "${arch}"; then
+            echo "${artifact} did not appear to be the correct architecture, check that Docker buildx is enabled"
+            exit 1
+        fi
     done
 done

aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/Dockerfile.agent

@@ -2,4 +2,4 @@ FROM public.ecr.aws/amazoncorretto/amazoncorretto:8
 
 RUN amazon-linux-extras enable docker && \
     yum clean metadata && \
-    yum install -y docker tar maven
+    yum install -y docker tar maven unzip file

aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.alpine.yml

@@ -39,7 +39,7 @@ phases:
       - (cd aws-lambda-java-events && mvn install)
       # Install serialization (dependency of RIC)
       - (cd aws-lambda-java-serialization && mvn install)
-      - (cd aws-lambda-java-runtime-interface-client && mvn install)
+      - (cd aws-lambda-java-runtime-interface-client && mvn install -DmultiArch=false)
       - (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install)
       - export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}"
       - echo "Extracting and including Runtime Interface Emulator"

aws-lambda-java-runtime-interface-client/test/integration/codebuild/buildspec.os.amazoncorretto.yml

@@ -37,7 +37,7 @@ phases:
       - (cd aws-lambda-java-events && mvn install)
       # Install serialization (dependency of RIC)
       - (cd aws-lambda-java-serialization && mvn install)
-      - (cd aws-lambda-java-runtime-interface-client && mvn install)
+      - (cd aws-lambda-java-runtime-interface-client && mvn install -DmultiArch=false)
       - (cd aws-lambda-java-runtime-interface-client/test/integration/test-handler && mvn install)
       - export IMAGE_TAG="java-${OS_DISTRIBUTION}-${DISTRO_VERSION}:${RUNTIME_VERSION}"
       - echo "Extracting and including Runtime Interface Emulator"