Initial commit

Author: rtyley

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on:
+  workflow_dispatch:
+  pull_request:
+  # Triggering CI on default branch improves caching
+  # see https://docs.github.com/en/free-pro-team@latest/actions/guides/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: corretto
+          java-version: 17
+          cache: sbt
+      - name: Find minimum memory required for the code to pass
+        run: ./findMinMem.sh

.gitignore

@@ -0,0 +1,9 @@
+target/
+
+.bsp/
+
+### IntelliJ IDEA ###
+.idea/
+
+### Mac OS ###
+.DS_Store

build.sbt

@@ -0,0 +1,18 @@
+Compile / run / fork := true
+
+// run / javaOptions += "-Xmx10m"
+
+libraryDependencies ++= Seq(
+  "software.amazon.awssdk" % "s3" % "2.20.120",
+  "software.amazon.awssdk" % "aws-crt-client" % "2.20.130",
+  "org.slf4j" % "slf4j-simple" % "2.0.7",
+  "com.google.guava" % "guava" % "32.1.2-jre"
+)
+
+commands += Command.single("runWithMaxMem") { (state, arg) =>
+  s"""set run / javaOptions += "-Xmx${arg}m"""" :: "run" :: state
+}
+
+commands += Command.single("repeatRunWithMaxMem") { (state, arg) =>
+   List.fill(20)(s"runWithMaxMem $arg") ::: state
+}

findMinMem.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+lo=1
+hi=1500
+while [ "$((hi - lo))" -gt 1 ]; do
+  mid=$((lo + (( hi - lo ) / 2)))
+  echo "Memory $mid MB"
+
+  if sbt "repeatRunWithMaxMem $mid";
+  then hi=$mid;
+  else lo=$mid; fi
+done
+echo "Failed with $lo MB, Succeeded with $hi MB"
+
+if [ -n "$GITHUB_STEP_SUMMARY" ]; then
+  echo "### Required $hi MB" >> "$GITHUB_STEP_SUMMARY"
+fi

project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.9.3

src/main/java/com/madgag/aws/sdk/async/responsebytes/KnownS3Object.java

@@ -0,0 +1,15 @@
+package com.madgag.aws.sdk.async.responsebytes;
+
+import com.google.common.hash.HashCode;
+
+public record KnownS3Object(String bucket, String key, HashCode sha256) {
+    public static KnownS3Object BIG = new KnownS3Object(
+            "sagemaker-sample-files", "datasets/tabular/customer-churn/customer-churn-data.zip",
+            HashCode.fromString("49e94915ca0de3368a0f715c088a38596c84f64a65f77d66c79c648f356c8008")
+    );
+
+    public static KnownS3Object SMALL = new KnownS3Object(
+            "sagemaker-sample-files", "datasets/tabular/xgb-churn/test-dataset.csv",
+            HashCode.fromString("1f395e2e95f746482989c9c943eabe0d9f6c69b505a05e067f6ae87e967aa0fc")
+    );
+}

src/main/java/com/madgag/aws/sdk/async/responsebytes/Main.java

@@ -0,0 +1,45 @@
+package com.madgag.aws.sdk.async.responsebytes;
+
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.core.ResponseBytes;
+import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.http.crt.AwsCrtAsyncHttpClient;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+
+import java.util.concurrent.ExecutionException;
+
+import static com.google.common.base.Verify.verify;
+import static java.time.Duration.ofSeconds;
+import static software.amazon.awssdk.regions.Region.US_EAST_1;
+
+public class Main {
+    public static void main(String[] args) throws ExecutionException, InterruptedException {
+        try (S3AsyncClient s3Client = buildS3Client()) {
+            System.out.println("About to fetch file...");
+            KnownS3Object knownS3Object = KnownS3Object.BIG;
+            ResponseBytes<GetObjectResponse> responseBytes = s3Client.getObject(
+                    GetObjectRequest.builder().bucket(knownS3Object.bucket()).key(knownS3Object.key()).build(),
+                    AsyncResponseTransformer.toBytes()
+            ).get();
+            System.out.println(responseBytes.response().contentLength());
+            HashCode hashOfDownloadedData = Hashing.sha256().hashBytes(responseBytes.asByteBuffer());
+            verify(hashOfDownloadedData.equals(knownS3Object.sha256()),
+                    "Downloaded hash (%s) doesn't match expected hash (%s)", hashOfDownloadedData, knownS3Object.sha256());
+        }
+    }
+
+    private static S3AsyncClient buildS3Client() {
+        return S3AsyncClient.builder()
+                .region(US_EAST_1)
+                .credentialsProvider(AnonymousCredentialsProvider.create())
+                .httpClientBuilder(AwsCrtAsyncHttpClient
+                        .builder()
+                        .connectionTimeout(ofSeconds(3))
+                        .maxConcurrency(100)).build();
+    }
+
+}