Add Endpoint to S3ClientConfiguration used to configure S3TransferManager

Author: ron1

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3ClientConfiguration.java

@@ -35,13 +35,15 @@
 @SdkPreviewApi
 public final class S3ClientConfiguration implements ToCopyableBuilder<S3ClientConfiguration.Builder, S3ClientConfiguration> {
     private final AwsCredentialsProvider credentialsProvider;
+    private final String endpoint;
     private final Region region;
     private final Long minimumPartSizeInBytes;
     private final Double targetThroughputInGbps;
     private final Integer maxConcurrency;
 
     private S3ClientConfiguration(DefaultBuilder builder) {
         this.credentialsProvider = builder.credentialsProvider;
+        this.endpoint = builder.endpoint;
         this.region = builder.region;
         this.minimumPartSizeInBytes = Validate.isPositiveOrNull(builder.minimumPartSizeInBytes, "minimumPartSizeInBytes");
         this.targetThroughputInGbps = Validate.isPositiveOrNull(builder.targetThroughputInGbps, "targetThroughputInGbps");
@@ -56,6 +58,13 @@ public Optional<AwsCredentialsProvider> credentialsProvider() {
         return Optional.ofNullable(credentialsProvider);
     }
 
+    /**
+     * @return the optional endpoint with which the SDK should communicate.
+     */
+    public Optional<String> endpoint() {
+        return Optional.ofNullable(endpoint);
+    }
+
     /**
      * @return the optional region with which the SDK should communicate.
      */
@@ -103,6 +112,9 @@ public boolean equals(Object o) {
         if (!Objects.equals(credentialsProvider, that.credentialsProvider)) {
             return false;
         }
+        if (!Objects.equals(endpoint, that.endpoint)) {
+            return false;
+        }
         if (!Objects.equals(region, that.region)) {
             return false;
         }
@@ -118,6 +130,7 @@ public boolean equals(Object o) {
     @Override
     public int hashCode() {
         int result = credentialsProvider != null ? credentialsProvider.hashCode() : 0;
+        result = 31 * result + (endpoint != null ? endpoint.hashCode() : 0);
         result = 31 * result + (region != null ? region.hashCode() : 0);
         result = 31 * result + (minimumPartSizeInBytes != null ? minimumPartSizeInBytes.hashCode() : 0);
         result = 31 * result + (targetThroughputInGbps != null ? targetThroughputInGbps.hashCode() : 0);
@@ -159,6 +172,16 @@ public interface Builder extends CopyableBuilder<Builder, S3ClientConfiguration>
          */
         Builder credentialsProvider(AwsCredentialsProvider credentialsProvider);
 
+        /**
+         * Configure the endpoint with which the SDK should communicate.
+         *
+         * <p>If this is not specified, the SDK will attempt to identify the endpoint automatically using the region parameter.
+         *
+         * @param endpoint the endpoint to be used
+         * @return this builder for method chaining.
+         */
+        Builder endpoint(String endpoint);
+
         /**
          * Configure the region with which the SDK should communicate.
          *
@@ -220,6 +243,7 @@ public interface Builder extends CopyableBuilder<Builder, S3ClientConfiguration>
 
     private static final class DefaultBuilder implements Builder {
         private AwsCredentialsProvider credentialsProvider;
+        private String endpoint;
         private Region region;
         private Long minimumPartSizeInBytes;
         private Double targetThroughputInGbps;
@@ -230,6 +254,7 @@ private DefaultBuilder() {
 
         private DefaultBuilder(S3ClientConfiguration configuration) {
             this.credentialsProvider = configuration.credentialsProvider;
+            this.endpoint = configuration.endpoint;
             this.region = configuration.region;
             this.minimumPartSizeInBytes = configuration.minimumPartSizeInBytes;
             this.targetThroughputInGbps = configuration.targetThroughputInGbps;
@@ -242,6 +267,12 @@ public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) {
             return this;
         }
 
+        @Override
+        public Builder endpoint(String endpoint) {
+            this.endpoint = endpoint;
+            return this;
+        }
+
         @Override
         public Builder region(Region region) {
             this.region = region;

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java

@@ -25,6 +25,8 @@
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import software.amazon.awssdk.core.async.AsyncResponseTransformer;
 import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration;
+import software.amazon.awssdk.crt.s3.S3Client;
+import software.amazon.awssdk.crt.s3.S3ClientOptions;
 import software.amazon.awssdk.http.SdkHttpResponse;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -42,6 +44,7 @@ public final class DefaultS3CrtAsyncClient implements S3CrtAsyncClient {
     public DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) {
         S3NativeClientConfiguration.Builder configBuilder =
             S3NativeClientConfiguration.builder()
+                                       .endpoint(builder.endpoint)
                                        .targetThroughputInGbps(builder.targetThroughputInGbps())
                                        .partSizeInBytes(builder.minimumPartSizeInBytes())
                                        .maxConcurrency(builder.maxConcurrency)
@@ -53,12 +56,16 @@ public DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) {
 
         configuration = configBuilder.build();
 
-        this.s3NativeClient = new S3NativeClient(configuration.signingRegion(),
-                                                 configuration.clientBootstrap(),
-                                                 configuration.credentialsProvider(),
-                                                 configuration.partSizeBytes(),
-                                                 configuration.targetThroughputInGbps(),
-                                                 configuration.maxConcurrency());
+        S3ClientOptions s3ClientOptions =
+            new S3ClientOptions().withEndpoint(configuration.endpoint())
+                                 .withRegion(configuration.signingRegion())
+                                 .withClientBootstrap(configuration.clientBootstrap())
+                                 .withCredentialsProvider(configuration.credentialsProvider())
+                                 .withPartSize(configuration.partSizeBytes())
+                                 .withThroughputTargetGbps(configuration.targetThroughputInGbps())
+                                 .withMaxConnections(configuration.maxConcurrency());
+
+        this.s3NativeClient = new S3NativeClient(configuration.signingRegion(), new S3Client(s3ClientOptions));
         this.crtErrorHandler = new CrtErrorHandler();
     }
 
@@ -157,6 +164,7 @@ public void close() {
 
     public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientBuilder {
         private AwsCredentialsProvider credentialsProvider;
+        private String endpoint;
         private Region region;
         private Long minimalPartSizeInBytes;
         private Double targetThroughputInGbps;
@@ -167,6 +175,10 @@ public AwsCredentialsProvider credentialsProvider() {
             return credentialsProvider;
         }
 
+        public String endpoint() {
+            return endpoint;
+        }
+
         public Region region() {
             return region;
         }
@@ -189,6 +201,12 @@ public S3CrtAsyncClientBuilder credentialsProvider(AwsCredentialsProvider creden
             return this;
         }
 
+        @Override
+        public S3CrtAsyncClientBuilder endpoint(String endpoint) {
+            this.endpoint = endpoint;
+            return this;
+        }
+
         @Override
         public S3CrtAsyncClientBuilder region(Region region) {
             this.region = region;

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java

@@ -81,6 +81,7 @@ private TransferManagerConfiguration resolveTransferManagerConfiguration(Default
     private S3CrtAsyncClient initializeS3CrtClient(DefaultBuilder tmBuilder) {
         S3CrtAsyncClient.S3CrtAsyncClientBuilder clientBuilder = S3CrtAsyncClient.builder();
         tmBuilder.s3ClientConfiguration.credentialsProvider().ifPresent(clientBuilder::credentialsProvider);
+        tmBuilder.s3ClientConfiguration.endpoint().ifPresent(clientBuilder::endpoint);
         tmBuilder.s3ClientConfiguration.maxConcurrency().ifPresent(clientBuilder::maxConcurrency);
         tmBuilder.s3ClientConfiguration.minimumPartSizeInBytes().ifPresent(clientBuilder::minimumPartSizeInBytes);
         tmBuilder.s3ClientConfiguration.region().ifPresent(clientBuilder::region);

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java

@@ -33,6 +33,8 @@ public interface S3CrtAsyncClient extends S3AsyncClient {
     interface S3CrtAsyncClientBuilder extends SdkBuilder<S3CrtAsyncClientBuilder, S3CrtAsyncClient> {
         S3CrtAsyncClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider);
 
+        S3CrtAsyncClientBuilder endpoint(String endpoint);
+
         S3CrtAsyncClientBuilder region(Region region);
 
         S3CrtAsyncClientBuilder minimumPartSizeInBytes(Long uploadPartSize);

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfiguration.java

@@ -43,6 +43,7 @@
 public class S3NativeClientConfiguration implements SdkAutoCloseable {
     private static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * SizeConstant.MB;
     private static final long DEFAULT_TARGET_THROUGHPUT_IN_GBPS = 5;
+    private final String endpoint;
     private final String signingRegion;
     private final ClientBootstrap clientBootstrap;
     private final CrtCredentialsProviderAdapter credentialProviderAdapter;
@@ -53,6 +54,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable {
     private final Executor futureCompletionExecutor;
 
     public S3NativeClientConfiguration(Builder builder) {
+        this.endpoint = builder.endpoint;
         this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() :
                              builder.signingRegion;
         this.clientBootstrap = new ClientBootstrap(null, null);
@@ -79,6 +81,10 @@ public static Builder builder() {
         return new Builder();
     }
 
+    public String endpoint() {
+        return endpoint;
+    }
+
     public String signingRegion() {
         return signingRegion;
     }
@@ -150,6 +156,7 @@ private void shutdownIfExecutorService(Object object) {
     }
 
     public static final class Builder {
+        private String endpoint;
         private String signingRegion;
         private AwsCredentialsProvider credentialsProvider;
         private Long partSizeInBytes;
@@ -160,6 +167,11 @@ public static final class Builder {
         private Builder() {
         }
 
+        public Builder endpoint(String endpoint) {
+            this.endpoint = endpoint;
+            return this;
+        }
+
         public Builder signingRegion(String signingRegion) {
             this.signingRegion = signingRegion;
             return this;

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java

@@ -72,12 +72,14 @@ public void build_allProperties() {
                                                                    .credentialsProvider(credentials)
                                                                    .maxConcurrency(100)
                                                                    .targetThroughputInGbps(10.0)
+                                                                   .endpoint("https://s3.us-west-2.amazonaws.com")
                                                                    .region(Region.US_WEST_2)
                                                                    .minimumPartSizeInBytes(5 * MB)
                                                                    .build();
 
         assertThat(configuration.credentialsProvider()).contains(credentials);
         assertThat(configuration.maxConcurrency()).contains(100);
+        assertThat(configuration.endpoint()).contains("https://s3.us-west-2.amazonaws.com");
         assertThat(configuration.region()).contains(Region.US_WEST_2);
         assertThat(configuration.targetThroughputInGbps()).contains(10.0);
         assertThat(configuration.minimumPartSizeInBytes()).contains(5 * MB);
@@ -90,6 +92,7 @@ public void build_emptyBuilder() {
 
         assertThat(configuration.credentialsProvider()).isEmpty();
         assertThat(configuration.maxConcurrency()).isEmpty();
+        assertThat(configuration.endpoint()).isEmpty();
         assertThat(configuration.region()).isEmpty();
         assertThat(configuration.targetThroughputInGbps()).isEmpty();
         assertThat(configuration.minimumPartSizeInBytes()).isEmpty();