Skip to content

Expose S3TransferManager property S3ClientConfiguration.endpointOverride #3084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 17, 2022
Merged

Expose S3TransferManager property S3ClientConfiguration.endpointOverride #3084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 31 additions & 1 deletion ...nsfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3ClientConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

package software.amazon.awssdk.transfer.s3;

import java.net.URI;
import java.util.Objects;
import java.util.Optional;
import software.amazon.awssdk.annotations.SdkPreviewApi;
Expand All @@ -39,6 +40,7 @@ public final class S3ClientConfiguration implements ToCopyableBuilder<S3ClientCo
private final Long minimumPartSizeInBytes;
private final Double targetThroughputInGbps;
private final Integer maxConcurrency;
private final URI endpointOverride;

private S3ClientConfiguration(DefaultBuilder builder) {
this.credentialsProvider = builder.credentialsProvider;
Expand All @@ -47,6 +49,7 @@ private S3ClientConfiguration(DefaultBuilder builder) {
this.targetThroughputInGbps = Validate.isPositiveOrNull(builder.targetThroughputInGbps, "targetThroughputInGbps");
this.maxConcurrency = Validate.isPositiveOrNull(builder.maxConcurrency,
"maxConcurrency");
this.endpointOverride = builder.endpointOverride;
}

/**
Expand Down Expand Up @@ -84,6 +87,13 @@ public Optional<Integer> maxConcurrency() {
return Optional.ofNullable(maxConcurrency);
}

/**
* @return the optional endpoint override with which the SDK should communicate.
*/
public Optional<URI> endpointOverride() {
return Optional.ofNullable(endpointOverride);
}

@Override
public Builder toBuilder() {
return new DefaultBuilder(this);
Expand Down Expand Up @@ -112,7 +122,10 @@ public boolean equals(Object o) {
if (!Objects.equals(targetThroughputInGbps, that.targetThroughputInGbps)) {
return false;
}
return Objects.equals(maxConcurrency, that.maxConcurrency);
if (!Objects.equals(maxConcurrency, that.maxConcurrency)) {
return false;
}
return Objects.equals(endpointOverride, that.endpointOverride);
}

@Override
Expand All @@ -122,6 +135,7 @@ public int hashCode() {
result = 31 * result + (minimumPartSizeInBytes != null ? minimumPartSizeInBytes.hashCode() : 0);
result = 31 * result + (targetThroughputInGbps != null ? targetThroughputInGbps.hashCode() : 0);
result = 31 * result + (maxConcurrency != null ? maxConcurrency.hashCode() : 0);
result = 31 * result + (endpointOverride != null ? endpointOverride.hashCode() : 0);
return result;
}

Expand Down Expand Up @@ -216,6 +230,14 @@ public interface Builder extends CopyableBuilder<Builder, S3ClientConfiguration>
* @see #targetThroughputInGbps(Double)
*/
Builder maxConcurrency(Integer maxConcurrency);

/**
* Configure the endpoint override with which the SDK should communicate.
*
* @param endpointOverride the endpoint override to be used
* @return this builder for method chaining.
*/
Builder endpointOverride(URI endpointOverride);
}

private static final class DefaultBuilder implements Builder {
Expand All @@ -224,6 +246,7 @@ private static final class DefaultBuilder implements Builder {
private Long minimumPartSizeInBytes;
private Double targetThroughputInGbps;
private Integer maxConcurrency;
private URI endpointOverride;

private DefaultBuilder() {
}
Expand All @@ -234,6 +257,7 @@ private DefaultBuilder(S3ClientConfiguration configuration) {
this.minimumPartSizeInBytes = configuration.minimumPartSizeInBytes;
this.targetThroughputInGbps = configuration.targetThroughputInGbps;
this.maxConcurrency = configuration.maxConcurrency;
this.endpointOverride = configuration.endpointOverride;
}

@Override
Expand Down Expand Up @@ -266,6 +290,12 @@ public Builder maxConcurrency(Integer maxConcurrency) {
return this;
}

@Override
public Builder endpointOverride(URI endpointOverride) {
this.endpointOverride = endpointOverride;
return this;
}

@Override
public S3ClientConfiguration build() {
return new S3ClientConfiguration(this);
Expand Down
15 changes: 15 additions & 0 deletions ...er/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.SDK_HTTP_EXECUTION_ATTRIBUTES;
import static software.amazon.awssdk.transfer.s3.internal.S3InternalSdkHttpExecutionAttribute.OPERATION_NAME;

import java.net.URI;
import java.util.concurrent.CompletableFuture;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.annotations.SdkTestInternalApi;
Expand Down Expand Up @@ -58,6 +59,7 @@ private DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) {
.minimumPartSizeInBytes(builder.minimumPartSizeInBytes())
.maxConcurrency(builder.maxConcurrency)
.region(builder.region)
.endpointOverride(builder.endpointOverride)
.credentialsProvider(builder.credentialsProvider)
.build();

Expand All @@ -79,6 +81,7 @@ private S3AsyncClient initializeS3AsyncClient(DefaultS3CrtClientBuilder builder)
.checksumValidationEnabled(false)
.build())
.region(builder.region)
.endpointOverride(builder.endpointOverride)
.credentialsProvider(builder.credentialsProvider)
.overrideConfiguration(o -> o.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
new NoOpSigner())
Expand Down Expand Up @@ -144,6 +147,7 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB
private Long minimalPartSizeInBytes;
private Double targetThroughputInGbps;
private Integer maxConcurrency;
private URI endpointOverride;

public AwsCredentialsProvider credentialsProvider() {
return credentialsProvider;
Expand All @@ -165,6 +169,11 @@ public Integer maxConcurrency() {
return maxConcurrency;
}

public URI endpointOverride() {
return endpointOverride;
}


@Override
public S3CrtAsyncClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider) {
this.credentialsProvider = credentialsProvider;
Expand Down Expand Up @@ -195,6 +204,12 @@ public S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency) {
return this;
}

@Override
public S3CrtAsyncClientBuilder endpointOverride(URI endpointOverride) {
this.endpointOverride = endpointOverride;
return this;
}

@Override
public S3CrtAsyncClient build() {
return new DefaultS3CrtAsyncClient(this);
Expand Down
1 change: 1 addition & 0 deletions ...r/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ private static S3CrtAsyncClient initializeS3CrtClient(DefaultBuilder tmBuilder)
tmBuilder.s3ClientConfiguration.minimumPartSizeInBytes().ifPresent(clientBuilder::minimumPartSizeInBytes);
tmBuilder.s3ClientConfiguration.region().ifPresent(clientBuilder::region);
tmBuilder.s3ClientConfiguration.targetThroughputInGbps().ifPresent(clientBuilder::targetThroughputInGbps);
tmBuilder.s3ClientConfiguration.endpointOverride().ifPresent(clientBuilder::endpointOverride);

return clientBuilder.build();
}
Expand Down
3 changes: 3 additions & 0 deletions ...r-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

package software.amazon.awssdk.transfer.s3.internal;

import java.net.URI;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.regions.Region;
Expand All @@ -39,6 +40,8 @@ interface S3CrtAsyncClientBuilder extends SdkBuilder<S3CrtAsyncClientBuilder, S3

S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency);

S3CrtAsyncClientBuilder endpointOverride(URI endpointOverride);

@Override
S3CrtAsyncClient build();
}
Expand Down
32 changes: 23 additions & 9 deletions ...nager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncHttpClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,14 @@ private S3CrtAsyncHttpClient(Builder builder) {
.partSizeInBytes(builder.minimalPartSizeInBytes)
.maxConcurrency(builder.maxConcurrency)
.signingRegion(builder.region == null ? null : builder.region.id())
.endpointOverride(builder.endpointOverride)
.credentialsProvider(builder.credentialsProvider)
.build();

S3ClientOptions s3ClientOptions =
new S3ClientOptions().withRegion(s3NativeClientConfiguration.signingRegion())
.withEndpoint(s3NativeClientConfiguration.endpointOverride() == null ? null :
s3NativeClientConfiguration.endpointOverride().toString())
.withCredentialsProvider(s3NativeClientConfiguration.credentialsProvider())
.withClientBootstrap(s3NativeClientConfiguration.clientBootstrap())
.withPartSize(s3NativeClientConfiguration.partSizeBytes())
Expand All @@ -76,17 +79,20 @@ private S3CrtAsyncHttpClient(Builder builder) {
}

@Override
public CompletableFuture<Void> execute(AsyncExecuteRequest request) {
public CompletableFuture<Void> execute(AsyncExecuteRequest asyncRequest) {
CompletableFuture<Void> executeFuture = new CompletableFuture<>();
HttpRequest httpRequest = toCrtRequest(request);
S3CrtResponseHandlerAdapter responseHandler = new S3CrtResponseHandlerAdapter(executeFuture, request.responseHandler());
URI uri = asyncRequest.request().getUri();
HttpRequest httpRequest = toCrtRequest(uri, asyncRequest);
S3CrtResponseHandlerAdapter responseHandler =
new S3CrtResponseHandlerAdapter(executeFuture, asyncRequest.responseHandler());

S3MetaRequestOptions.MetaRequestType requestType = requestType(request);
S3MetaRequestOptions.MetaRequestType requestType = requestType(asyncRequest);

S3MetaRequestOptions requestOptions = new S3MetaRequestOptions()
.withHttpRequest(httpRequest)
.withMetaRequestType(requestType)
.withResponseHandler(responseHandler);
.withResponseHandler(responseHandler)
.withEndpoint(s3NativeClientConfiguration.endpointOverride());

try (S3MetaRequest s3MetaRequest = crtS3Client.makeMetaRequest(requestOptions)) {
closeResourcesWhenComplete(executeFuture, s3MetaRequest);
Expand All @@ -100,8 +106,8 @@ public String clientName() {
return "s3crt";
}

private static S3MetaRequestOptions.MetaRequestType requestType(AsyncExecuteRequest request) {
String operationName = request.httpExecutionAttributes().getAttribute(OPERATION_NAME);
private static S3MetaRequestOptions.MetaRequestType requestType(AsyncExecuteRequest asyncRequest) {
String operationName = asyncRequest.httpExecutionAttributes().getAttribute(OPERATION_NAME);
if (operationName != null) {
switch (operationName) {
case "GetObject":
Expand All @@ -128,8 +134,7 @@ private static void closeResourcesWhenComplete(CompletableFuture<Void> executeFu
});
}

private static HttpRequest toCrtRequest(AsyncExecuteRequest asyncRequest) {
URI uri = asyncRequest.request().getUri();
private static HttpRequest toCrtRequest(URI uri, AsyncExecuteRequest asyncRequest) {
SdkHttpRequest sdkRequest = asyncRequest.request();

String method = sdkRequest.method().name();
Expand Down Expand Up @@ -166,6 +171,7 @@ public static final class Builder implements SdkAsyncHttpClient.Builder<S3CrtAsy
private Long minimalPartSizeInBytes;
private Double targetThroughputInGbps;
private Integer maxConcurrency;
private URI endpointOverride;

/**
* Configure the credentials that should be used to authenticate with S3.
Expand Down Expand Up @@ -221,6 +227,14 @@ public Builder maxConcurrency(Integer maxConcurrency) {
return this;
}

/**
* Configure the endpoint override with which the SDK should communicate.
*/
public Builder endpointOverride(URI endpointOverride) {
this.endpointOverride = endpointOverride;
return this;
}

@Override
public SdkAsyncHttpClient build() {
return new S3CrtAsyncHttpClient(this);
Expand Down
14 changes: 14 additions & 0 deletions ...rc/main/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import static software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR;

import java.net.URI;
import java.util.Optional;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
Expand Down Expand Up @@ -50,6 +51,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable {
private final long partSizeInBytes;
private final double targetThroughputInGbps;
private final int maxConcurrency;
private final URI endpointOverride;
private final Executor futureCompletionExecutor;

public S3NativeClientConfiguration(Builder builder) {
Expand All @@ -72,6 +74,8 @@ public S3NativeClientConfiguration(Builder builder) {
// Using 0 so that CRT will calculate it based on targetThroughputGbps
this.maxConcurrency = builder.maxConcurrency == null ? 0 : builder.maxConcurrency;

this.endpointOverride = builder.endpointOverride;

this.futureCompletionExecutor = resolveAsyncFutureCompletionExecutor(builder.asynConfiguration);
}

Expand Down Expand Up @@ -103,6 +107,10 @@ public int maxConcurrency() {
return maxConcurrency;
}

public URI endpointOverride() {
return endpointOverride;
}

public Executor futureCompletionExecutor() {
return futureCompletionExecutor;
}
Expand Down Expand Up @@ -155,6 +163,7 @@ public static final class Builder {
private Long partSizeInBytes;
private Double targetThroughputInGbps;
private Integer maxConcurrency;
private URI endpointOverride;
private ClientAsyncConfiguration asynConfiguration;

private Builder() {
Expand Down Expand Up @@ -185,6 +194,11 @@ public Builder maxConcurrency(Integer maxConcurrency) {
return this;
}

public Builder endpointOverride(URI endpointOverride) {
this.endpointOverride = endpointOverride;
return this;
}

public Builder asyncConfiguration(ClientAsyncConfiguration asyncConfiguration) {
this.asynConfiguration = asyncConfiguration;
return this;
Expand Down
5 changes: 5 additions & 0 deletions ...r-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static software.amazon.awssdk.transfer.s3.SizeConstant.MB;

import java.net.URI;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
Expand Down Expand Up @@ -73,12 +74,15 @@ public void build_allProperties() {
.maxConcurrency(100)
.targetThroughputInGbps(10.0)
.region(Region.US_WEST_2)
.endpointOverride(URI.create(
"http://s3.us-west-1.amazonaws.com:80"))
.minimumPartSizeInBytes(5 * MB)
.build();

assertThat(configuration.credentialsProvider()).contains(credentials);
assertThat(configuration.maxConcurrency()).contains(100);
assertThat(configuration.region()).contains(Region.US_WEST_2);
assertThat(configuration.endpointOverride().toString()).contains("http://s3.us-west-1.amazonaws.com:80");
assertThat(configuration.targetThroughputInGbps()).contains(10.0);
assertThat(configuration.minimumPartSizeInBytes()).contains(5 * MB);
}
Expand All @@ -91,6 +95,7 @@ public void build_emptyBuilder() {
assertThat(configuration.credentialsProvider()).isEmpty();
assertThat(configuration.maxConcurrency()).isEmpty();
assertThat(configuration.region()).isEmpty();
assertThat(configuration.endpointOverride()).isEmpty();
assertThat(configuration.targetThroughputInGbps()).isEmpty();
assertThat(configuration.minimumPartSizeInBytes()).isEmpty();
}
Expand Down
Loading