Skip to content

Commit 7121f4f

Browse files
ron1ron1
authored
Expose S3TransferManager config property S3ClientConfiguration.endpointOverride (#3084)
1 parent de37830 commit 7121f4f

File tree

8 files changed

+109
-19
lines changed

8 files changed

+109
-19
lines changed

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3ClientConfiguration.java

+31-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
package software.amazon.awssdk.transfer.s3;
1717

18+
import java.net.URI;
1819
import java.util.Objects;
1920
import java.util.Optional;
2021
import software.amazon.awssdk.annotations.SdkPreviewApi;
@@ -39,6 +40,7 @@ public final class S3ClientConfiguration implements ToCopyableBuilder<S3ClientCo
3940
private final Long minimumPartSizeInBytes;
4041
private final Double targetThroughputInGbps;
4142
private final Integer maxConcurrency;
43+
private final URI endpointOverride;
4244

4345
private S3ClientConfiguration(DefaultBuilder builder) {
4446
this.credentialsProvider = builder.credentialsProvider;
@@ -47,6 +49,7 @@ private S3ClientConfiguration(DefaultBuilder builder) {
4749
this.targetThroughputInGbps = Validate.isPositiveOrNull(builder.targetThroughputInGbps, "targetThroughputInGbps");
4850
this.maxConcurrency = Validate.isPositiveOrNull(builder.maxConcurrency,
4951
"maxConcurrency");
52+
this.endpointOverride = builder.endpointOverride;
5053
}
5154

5255
/**
@@ -84,6 +87,13 @@ public Optional<Integer> maxConcurrency() {
8487
return Optional.ofNullable(maxConcurrency);
8588
}
8689

90+
/**
91+
* @return the optional endpoint override with which the SDK should communicate.
92+
*/
93+
public Optional<URI> endpointOverride() {
94+
return Optional.ofNullable(endpointOverride);
95+
}
96+
8797
@Override
8898
public Builder toBuilder() {
8999
return new DefaultBuilder(this);
@@ -112,7 +122,10 @@ public boolean equals(Object o) {
112122
if (!Objects.equals(targetThroughputInGbps, that.targetThroughputInGbps)) {
113123
return false;
114124
}
115-
return Objects.equals(maxConcurrency, that.maxConcurrency);
125+
if (!Objects.equals(maxConcurrency, that.maxConcurrency)) {
126+
return false;
127+
}
128+
return Objects.equals(endpointOverride, that.endpointOverride);
116129
}
117130

118131
@Override
@@ -122,6 +135,7 @@ public int hashCode() {
122135
result = 31 * result + (minimumPartSizeInBytes != null ? minimumPartSizeInBytes.hashCode() : 0);
123136
result = 31 * result + (targetThroughputInGbps != null ? targetThroughputInGbps.hashCode() : 0);
124137
result = 31 * result + (maxConcurrency != null ? maxConcurrency.hashCode() : 0);
138+
result = 31 * result + (endpointOverride != null ? endpointOverride.hashCode() : 0);
125139
return result;
126140
}
127141

@@ -216,6 +230,14 @@ public interface Builder extends CopyableBuilder<Builder, S3ClientConfiguration>
216230
* @see #targetThroughputInGbps(Double)
217231
*/
218232
Builder maxConcurrency(Integer maxConcurrency);
233+
234+
/**
235+
* Configure the endpoint override with which the SDK should communicate.
236+
*
237+
* @param endpointOverride the endpoint override to be used
238+
* @return this builder for method chaining.
239+
*/
240+
Builder endpointOverride(URI endpointOverride);
219241
}
220242

221243
private static final class DefaultBuilder implements Builder {
@@ -224,6 +246,7 @@ private static final class DefaultBuilder implements Builder {
224246
private Long minimumPartSizeInBytes;
225247
private Double targetThroughputInGbps;
226248
private Integer maxConcurrency;
249+
private URI endpointOverride;
227250

228251
private DefaultBuilder() {
229252
}
@@ -234,6 +257,7 @@ private DefaultBuilder(S3ClientConfiguration configuration) {
234257
this.minimumPartSizeInBytes = configuration.minimumPartSizeInBytes;
235258
this.targetThroughputInGbps = configuration.targetThroughputInGbps;
236259
this.maxConcurrency = configuration.maxConcurrency;
260+
this.endpointOverride = configuration.endpointOverride;
237261
}
238262

239263
@Override
@@ -266,6 +290,12 @@ public Builder maxConcurrency(Integer maxConcurrency) {
266290
return this;
267291
}
268292

293+
@Override
294+
public Builder endpointOverride(URI endpointOverride) {
295+
this.endpointOverride = endpointOverride;
296+
return this;
297+
}
298+
269299
@Override
270300
public S3ClientConfiguration build() {
271301
return new S3ClientConfiguration(this);

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java

+15
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.SDK_HTTP_EXECUTION_ATTRIBUTES;
1919
import static software.amazon.awssdk.transfer.s3.internal.S3InternalSdkHttpExecutionAttribute.OPERATION_NAME;
2020

21+
import java.net.URI;
2122
import java.util.concurrent.CompletableFuture;
2223
import software.amazon.awssdk.annotations.SdkInternalApi;
2324
import software.amazon.awssdk.annotations.SdkTestInternalApi;
@@ -58,6 +59,7 @@ private DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) {
5859
.minimumPartSizeInBytes(builder.minimumPartSizeInBytes())
5960
.maxConcurrency(builder.maxConcurrency)
6061
.region(builder.region)
62+
.endpointOverride(builder.endpointOverride)
6163
.credentialsProvider(builder.credentialsProvider)
6264
.build();
6365

@@ -79,6 +81,7 @@ private S3AsyncClient initializeS3AsyncClient(DefaultS3CrtClientBuilder builder)
7981
.checksumValidationEnabled(false)
8082
.build())
8183
.region(builder.region)
84+
.endpointOverride(builder.endpointOverride)
8285
.credentialsProvider(builder.credentialsProvider)
8386
.overrideConfiguration(o -> o.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
8487
new NoOpSigner())
@@ -144,6 +147,7 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB
144147
private Long minimalPartSizeInBytes;
145148
private Double targetThroughputInGbps;
146149
private Integer maxConcurrency;
150+
private URI endpointOverride;
147151

148152
public AwsCredentialsProvider credentialsProvider() {
149153
return credentialsProvider;
@@ -165,6 +169,11 @@ public Integer maxConcurrency() {
165169
return maxConcurrency;
166170
}
167171

172+
public URI endpointOverride() {
173+
return endpointOverride;
174+
}
175+
176+
168177
@Override
169178
public S3CrtAsyncClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider) {
170179
this.credentialsProvider = credentialsProvider;
@@ -195,6 +204,12 @@ public S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency) {
195204
return this;
196205
}
197206

207+
@Override
208+
public S3CrtAsyncClientBuilder endpointOverride(URI endpointOverride) {
209+
this.endpointOverride = endpointOverride;
210+
return this;
211+
}
212+
198213
@Override
199214
public S3CrtAsyncClient build() {
200215
return new DefaultS3CrtAsyncClient(this);

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java

+1
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ private static S3CrtAsyncClient initializeS3CrtClient(DefaultBuilder tmBuilder)
9696
tmBuilder.s3ClientConfiguration.minimumPartSizeInBytes().ifPresent(clientBuilder::minimumPartSizeInBytes);
9797
tmBuilder.s3ClientConfiguration.region().ifPresent(clientBuilder::region);
9898
tmBuilder.s3ClientConfiguration.targetThroughputInGbps().ifPresent(clientBuilder::targetThroughputInGbps);
99+
tmBuilder.s3ClientConfiguration.endpointOverride().ifPresent(clientBuilder::endpointOverride);
99100

100101
return clientBuilder.build();
101102
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java

+3
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
package software.amazon.awssdk.transfer.s3.internal;
1717

18+
import java.net.URI;
1819
import software.amazon.awssdk.annotations.SdkInternalApi;
1920
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
2021
import software.amazon.awssdk.regions.Region;
@@ -39,6 +40,8 @@ interface S3CrtAsyncClientBuilder extends SdkBuilder<S3CrtAsyncClientBuilder, S3
3940

4041
S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency);
4142

43+
S3CrtAsyncClientBuilder endpointOverride(URI endpointOverride);
44+
4245
@Override
4346
S3CrtAsyncClient build();
4447
}

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncHttpClient.java

+23-9
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,14 @@ private S3CrtAsyncHttpClient(Builder builder) {
5757
.partSizeInBytes(builder.minimalPartSizeInBytes)
5858
.maxConcurrency(builder.maxConcurrency)
5959
.signingRegion(builder.region == null ? null : builder.region.id())
60+
.endpointOverride(builder.endpointOverride)
6061
.credentialsProvider(builder.credentialsProvider)
6162
.build();
6263

6364
S3ClientOptions s3ClientOptions =
6465
new S3ClientOptions().withRegion(s3NativeClientConfiguration.signingRegion())
66+
.withEndpoint(s3NativeClientConfiguration.endpointOverride() == null ? null :
67+
s3NativeClientConfiguration.endpointOverride().toString())
6568
.withCredentialsProvider(s3NativeClientConfiguration.credentialsProvider())
6669
.withClientBootstrap(s3NativeClientConfiguration.clientBootstrap())
6770
.withPartSize(s3NativeClientConfiguration.partSizeBytes())
@@ -76,17 +79,20 @@ private S3CrtAsyncHttpClient(Builder builder) {
7679
}
7780

7881
@Override
79-
public CompletableFuture<Void> execute(AsyncExecuteRequest request) {
82+
public CompletableFuture<Void> execute(AsyncExecuteRequest asyncRequest) {
8083
CompletableFuture<Void> executeFuture = new CompletableFuture<>();
81-
HttpRequest httpRequest = toCrtRequest(request);
82-
S3CrtResponseHandlerAdapter responseHandler = new S3CrtResponseHandlerAdapter(executeFuture, request.responseHandler());
84+
URI uri = asyncRequest.request().getUri();
85+
HttpRequest httpRequest = toCrtRequest(uri, asyncRequest);
86+
S3CrtResponseHandlerAdapter responseHandler =
87+
new S3CrtResponseHandlerAdapter(executeFuture, asyncRequest.responseHandler());
8388

84-
S3MetaRequestOptions.MetaRequestType requestType = requestType(request);
89+
S3MetaRequestOptions.MetaRequestType requestType = requestType(asyncRequest);
8590

8691
S3MetaRequestOptions requestOptions = new S3MetaRequestOptions()
8792
.withHttpRequest(httpRequest)
8893
.withMetaRequestType(requestType)
89-
.withResponseHandler(responseHandler);
94+
.withResponseHandler(responseHandler)
95+
.withEndpoint(s3NativeClientConfiguration.endpointOverride());
9096

9197
try (S3MetaRequest s3MetaRequest = crtS3Client.makeMetaRequest(requestOptions)) {
9298
closeResourcesWhenComplete(executeFuture, s3MetaRequest);
@@ -100,8 +106,8 @@ public String clientName() {
100106
return "s3crt";
101107
}
102108

103-
private static S3MetaRequestOptions.MetaRequestType requestType(AsyncExecuteRequest request) {
104-
String operationName = request.httpExecutionAttributes().getAttribute(OPERATION_NAME);
109+
private static S3MetaRequestOptions.MetaRequestType requestType(AsyncExecuteRequest asyncRequest) {
110+
String operationName = asyncRequest.httpExecutionAttributes().getAttribute(OPERATION_NAME);
105111
if (operationName != null) {
106112
switch (operationName) {
107113
case "GetObject":
@@ -128,8 +134,7 @@ private static void closeResourcesWhenComplete(CompletableFuture<Void> executeFu
128134
});
129135
}
130136

131-
private static HttpRequest toCrtRequest(AsyncExecuteRequest asyncRequest) {
132-
URI uri = asyncRequest.request().getUri();
137+
private static HttpRequest toCrtRequest(URI uri, AsyncExecuteRequest asyncRequest) {
133138
SdkHttpRequest sdkRequest = asyncRequest.request();
134139

135140
String method = sdkRequest.method().name();
@@ -166,6 +171,7 @@ public static final class Builder implements SdkAsyncHttpClient.Builder<S3CrtAsy
166171
private Long minimalPartSizeInBytes;
167172
private Double targetThroughputInGbps;
168173
private Integer maxConcurrency;
174+
private URI endpointOverride;
169175

170176
/**
171177
* Configure the credentials that should be used to authenticate with S3.
@@ -221,6 +227,14 @@ public Builder maxConcurrency(Integer maxConcurrency) {
221227
return this;
222228
}
223229

230+
/**
231+
* Configure the endpoint override with which the SDK should communicate.
232+
*/
233+
public Builder endpointOverride(URI endpointOverride) {
234+
this.endpointOverride = endpointOverride;
235+
return this;
236+
}
237+
224238
@Override
225239
public SdkAsyncHttpClient build() {
226240
return new S3CrtAsyncHttpClient(this);

‎services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfiguration.java

+14
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import static software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR;
2020

21+
import java.net.URI;
2122
import java.util.Optional;
2223
import java.util.concurrent.Executor;
2324
import java.util.concurrent.ExecutorService;
@@ -50,6 +51,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable {
5051
private final long partSizeInBytes;
5152
private final double targetThroughputInGbps;
5253
private final int maxConcurrency;
54+
private final URI endpointOverride;
5355
private final Executor futureCompletionExecutor;
5456

5557
public S3NativeClientConfiguration(Builder builder) {
@@ -72,6 +74,8 @@ public S3NativeClientConfiguration(Builder builder) {
7274
// Using 0 so that CRT will calculate it based on targetThroughputGbps
7375
this.maxConcurrency = builder.maxConcurrency == null ? 0 : builder.maxConcurrency;
7476

77+
this.endpointOverride = builder.endpointOverride;
78+
7579
this.futureCompletionExecutor = resolveAsyncFutureCompletionExecutor(builder.asynConfiguration);
7680
}
7781

@@ -103,6 +107,10 @@ public int maxConcurrency() {
103107
return maxConcurrency;
104108
}
105109

110+
public URI endpointOverride() {
111+
return endpointOverride;
112+
}
113+
106114
public Executor futureCompletionExecutor() {
107115
return futureCompletionExecutor;
108116
}
@@ -155,6 +163,7 @@ public static final class Builder {
155163
private Long partSizeInBytes;
156164
private Double targetThroughputInGbps;
157165
private Integer maxConcurrency;
166+
private URI endpointOverride;
158167
private ClientAsyncConfiguration asynConfiguration;
159168

160169
private Builder() {
@@ -185,6 +194,11 @@ public Builder maxConcurrency(Integer maxConcurrency) {
185194
return this;
186195
}
187196

197+
public Builder endpointOverride(URI endpointOverride) {
198+
this.endpointOverride = endpointOverride;
199+
return this;
200+
}
201+
188202
public Builder asyncConfiguration(ClientAsyncConfiguration asyncConfiguration) {
189203
this.asynConfiguration = asyncConfiguration;
190204
return this;

‎services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java

+5
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static org.assertj.core.api.Assertions.assertThatThrownBy;
2020
import static software.amazon.awssdk.transfer.s3.SizeConstant.MB;
2121

22+
import java.net.URI;
2223
import nl.jqno.equalsverifier.EqualsVerifier;
2324
import org.junit.jupiter.api.Test;
2425
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
@@ -73,12 +74,15 @@ public void build_allProperties() {
7374
.maxConcurrency(100)
7475
.targetThroughputInGbps(10.0)
7576
.region(Region.US_WEST_2)
77+
.endpointOverride(URI.create(
78+
"http://s3.us-west-1.amazonaws.com:80"))
7679
.minimumPartSizeInBytes(5 * MB)
7780
.build();
7881

7982
assertThat(configuration.credentialsProvider()).contains(credentials);
8083
assertThat(configuration.maxConcurrency()).contains(100);
8184
assertThat(configuration.region()).contains(Region.US_WEST_2);
85+
assertThat(configuration.endpointOverride().toString()).contains("http://s3.us-west-1.amazonaws.com:80");
8286
assertThat(configuration.targetThroughputInGbps()).contains(10.0);
8387
assertThat(configuration.minimumPartSizeInBytes()).contains(5 * MB);
8488
}
@@ -91,6 +95,7 @@ public void build_emptyBuilder() {
9195
assertThat(configuration.credentialsProvider()).isEmpty();
9296
assertThat(configuration.maxConcurrency()).isEmpty();
9397
assertThat(configuration.region()).isEmpty();
98+
assertThat(configuration.endpointOverride()).isEmpty();
9499
assertThat(configuration.targetThroughputInGbps()).isEmpty();
95100
assertThat(configuration.minimumPartSizeInBytes()).isEmpty();
96101
}

0 commit comments

Comments
 (0)