diff --git a/.changes/next-release/bugfix-AWSSDKforJavav2-0da2191.json b/.changes/next-release/bugfix-AWSSDKforJavav2-0da2191.json new file mode 100644 index 000000000000..bc2ddf2bb4a2 --- /dev/null +++ b/.changes/next-release/bugfix-AWSSDKforJavav2-0da2191.json @@ -0,0 +1,6 @@ +{ + "type": "bugfix", + "category": "AWS SDK for Java v2", + "contributor": "", + "description": "Fix issue in `FileAsyncRequestBody` where the underlying file channel would only be closed when enough requests are sent to read *past* the end of the file; if just enough requests are sent to read to the end of the file, the file is not closed, leaving an open file descriptor around longer than it needs to be." +} diff --git a/.changes/next-release/feature-AmazonS3TransferManagerPreview-e951466.json b/.changes/next-release/feature-AmazonS3TransferManagerPreview-e951466.json new file mode 100644 index 000000000000..b1466d621239 --- /dev/null +++ b/.changes/next-release/feature-AmazonS3TransferManagerPreview-e951466.json @@ -0,0 +1,6 @@ +{ + "category": "Amazon S3 Transfer Manager [Preview]", + "contributor": "", + "type": "feature", + "description": "This release includes the preview release of the Amazon S3 Transfer Manager. Visit our [Developer Guide](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/transfer-manager.html) for more information." +} diff --git a/buildspecs/release-to-maven.yml b/buildspecs/release-to-maven.yml index 43cc424aa525..e734173cbaa3 100644 --- a/buildspecs/release-to-maven.yml +++ b/buildspecs/release-to-maven.yml @@ -26,7 +26,7 @@ phases: if ! curl -f --head $SONATYPE_URL; then mkdir -p $CREDENTIALS aws s3 cp s3://aws-java-sdk-release-credentials/ $CREDENTIALS/ --recursive - mvn clean deploy -B -s $SETTINGS_XML -Dgpg.homedir=$GPG_HOME -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-sts-testing -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30 + mvn clean deploy -B -s $SETTINGS_XML -Dgpg.homedir=$GPG_HOME -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-sts-testing,!:s3-benchmarks -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30 else echo "This version was already released." fi diff --git a/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProvider.java b/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProvider.java index 5ae570867730..56bfdb2b03ba 100644 --- a/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProvider.java +++ b/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProvider.java @@ -24,7 +24,7 @@ import software.amazon.awssdk.core.SdkSystemSetting; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkServiceException; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.regions.util.HttpResourcesUtils; import software.amazon.awssdk.regions.util.ResourcesEndpointProvider; import software.amazon.awssdk.utils.ToString; @@ -160,7 +160,7 @@ public URI endpoint() throws IOException { @Override public Map headers() { Map requestHeaders = new HashMap<>(); - requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent()); + requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent()); requestHeaders.put("Accept", "*/*"); requestHeaders.put("Connection", "keep-alive"); @@ -195,7 +195,7 @@ public URI endpoint() { @Override public Map headers() { Map requestHeaders = new HashMap<>(); - requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent()); + requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent()); requestHeaders.put("Accept", "*/*"); requestHeaders.put("Connection", "keep-alive"); requestHeaders.put(EC2_METADATA_TOKEN_TTL_HEADER, DEFAULT_TOKEN_TTL); diff --git a/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ContainerCredentialsProviderTest.java b/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ContainerCredentialsProviderTest.java index 507b855d3831..517edb21a6d9 100644 --- a/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ContainerCredentialsProviderTest.java +++ b/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ContainerCredentialsProviderTest.java @@ -33,7 +33,7 @@ import org.junit.ClassRule; import org.junit.Test; import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.regions.util.ResourcesEndpointProvider; import software.amazon.awssdk.testutils.EnvironmentVariableHelper; @@ -122,7 +122,7 @@ private void stubForCorruptedSuccessResponse() { private void stubFor200Response(String body) { stubFor(get(urlPathEqualTo(CREDENTIALS_PATH)) - .withHeader("User-Agent", equalTo(UserAgentUtils.getUserAgent())) + .withHeader("User-Agent", equalTo(SdkUserAgent.create().userAgent())) .willReturn(aResponse() .withStatus(200) .withHeader("Content-Type", "application/json") diff --git a/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/HttpCredentialsProviderTest.java b/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/HttpCredentialsProviderTest.java index 3e16b3dd3f05..2d54507d5ca5 100644 --- a/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/HttpCredentialsProviderTest.java +++ b/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/HttpCredentialsProviderTest.java @@ -35,7 +35,7 @@ import org.junit.ClassRule; import org.junit.Test; import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.regions.util.ResourcesEndpointProvider; import software.amazon.awssdk.utils.DateUtils; import software.amazon.awssdk.utils.IoUtils; @@ -132,7 +132,7 @@ public void basicCachingFunctionalityWorks() { private void stubForSuccessResponseWithCustomBody(String body) { stubFor( get(urlPathEqualTo(CREDENTIALS_PATH)) - .withHeader("User-Agent", equalTo(UserAgentUtils.getUserAgent())) + .withHeader("User-Agent", equalTo(SdkUserAgent.create().userAgent())) .willReturn(aResponse() .withStatus(200) .withHeader("Content-Type", "application/json") diff --git a/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProviderTest.java b/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProviderTest.java index 8b699b7c3ac7..05b4fb6f5885 100644 --- a/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProviderTest.java +++ b/core/auth/src/test/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProviderTest.java @@ -23,14 +23,11 @@ import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; -import static org.assertj.core.api.Assertions.assertThat; -import static org.hamcrest.Matchers.instanceOf; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; -import java.net.SocketTimeoutException; import java.time.Duration; import java.time.Instant; import org.junit.AfterClass; @@ -40,8 +37,7 @@ import org.junit.rules.ExpectedException; import software.amazon.awssdk.core.SdkSystemSetting; import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; -import software.amazon.awssdk.profiles.ProfileFileSystemSetting; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.utils.DateUtils; public class InstanceProfileCredentialsProviderTest { @@ -94,7 +90,7 @@ public void resolveCredentials_requestsIncludeUserAgent() { provider.resolveCredentials(); String userAgentHeader = "User-Agent"; - String userAgent = UserAgentUtils.getUserAgent(); + String userAgent = SdkUserAgent.create().userAgent(); WireMock.verify(putRequestedFor(urlPathEqualTo(TOKEN_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent))); WireMock.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent))); WireMock.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH + "some-profile")).withHeader(userAgentHeader, equalTo(userAgent))); @@ -235,7 +231,7 @@ public void resolveCredentials_customProfileFileAndName_usesCorrectEndpoint() { provider.resolveCredentials(); String userAgentHeader = "User-Agent"; - String userAgent = UserAgentUtils.getUserAgent(); + String userAgent = SdkUserAgent.create().userAgent(); mockMetadataEndpoint_2.verify(putRequestedFor(urlPathEqualTo(TOKEN_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent))); mockMetadataEndpoint_2.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent))); mockMetadataEndpoint_2.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH + "some-profile")).withHeader(userAgentHeader, equalTo(userAgent))); diff --git a/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java b/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java index e6ba2ff54b3a..13a2dbb245eb 100644 --- a/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java +++ b/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java @@ -32,7 +32,7 @@ import software.amazon.awssdk.core.SdkSystemSetting; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkServiceException; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.protocols.jsoncore.JsonNode; import software.amazon.awssdk.protocols.jsoncore.JsonNodeParser; import software.amazon.awssdk.regions.util.HttpResourcesUtils; @@ -633,7 +633,7 @@ public URI endpoint() { @Override public Map headers() { Map requestHeaders = new HashMap<>(); - requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent()); + requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent()); requestHeaders.put("Accept", "*/*"); requestHeaders.put("Connection", "keep-alive"); diff --git a/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/InstanceProviderTokenEndpointProvider.java b/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/InstanceProviderTokenEndpointProvider.java index b8d1e7c0e4ba..0eb655cac855 100644 --- a/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/InstanceProviderTokenEndpointProvider.java +++ b/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/InstanceProviderTokenEndpointProvider.java @@ -19,7 +19,7 @@ import java.util.HashMap; import java.util.Map; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.regions.util.ResourcesEndpointProvider; @SdkInternalApi @@ -43,7 +43,7 @@ public URI endpoint() { @Override public Map headers() { Map requestHeaders = new HashMap<>(); - requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent()); + requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent()); requestHeaders.put("Accept", "*/*"); requestHeaders.put("Connection", "keep-alive"); requestHeaders.put(EC2_METADATA_TOKEN_TTL_HEADER, DEFAULT_TOKEN_TTL); diff --git a/core/regions/src/main/java/software/amazon/awssdk/regions/util/ResourcesEndpointProvider.java b/core/regions/src/main/java/software/amazon/awssdk/regions/util/ResourcesEndpointProvider.java index 7344595d8c19..6005ea4bde6d 100644 --- a/core/regions/src/main/java/software/amazon/awssdk/regions/util/ResourcesEndpointProvider.java +++ b/core/regions/src/main/java/software/amazon/awssdk/regions/util/ResourcesEndpointProvider.java @@ -20,7 +20,7 @@ import java.util.HashMap; import java.util.Map; import software.amazon.awssdk.annotations.SdkProtectedApi; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; /** *

@@ -57,7 +57,7 @@ default ResourcesEndpointRetryPolicy retryPolicy() { */ default Map headers() { Map requestHeaders = new HashMap<>(); - requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent()); + requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent()); requestHeaders.put("Accept", "*/*"); requestHeaders.put("Connection", "keep-alive"); diff --git a/core/regions/src/test/java/software/amazon/awssdk/regions/util/HttpCredentialsUtilsTest.java b/core/regions/src/test/java/software/amazon/awssdk/regions/util/HttpCredentialsUtilsTest.java index e96d4a9174c0..5925d3f623f3 100644 --- a/core/regions/src/test/java/software/amazon/awssdk/regions/util/HttpCredentialsUtilsTest.java +++ b/core/regions/src/test/java/software/amazon/awssdk/regions/util/HttpCredentialsUtilsTest.java @@ -36,8 +36,7 @@ import org.mockito.runners.MockitoJUnitRunner; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkServiceException; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; -import software.amazon.awssdk.core.util.VersionInfo; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.regions.internal.util.ConnectionUtils; import software.amazon.awssdk.regions.internal.util.SocketUtils; @@ -53,7 +52,7 @@ public class HttpCredentialsUtilsTest { private static Map headers = new HashMap() { { - put("User-Agent", UserAgentUtils.getUserAgent()); + put("User-Agent", SdkUserAgent.create().userAgent()); put("Accept", "*/*"); put("Connection", "keep-alive"); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/builder/SdkDefaultClientBuilder.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/builder/SdkDefaultClientBuilder.java index 30af786e5ecc..cade6174232b 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/builder/SdkDefaultClientBuilder.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/builder/SdkDefaultClientBuilder.java @@ -63,9 +63,9 @@ import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; import software.amazon.awssdk.core.internal.http.loader.DefaultSdkAsyncHttpClientBuilder; import software.amazon.awssdk.core.internal.http.loader.DefaultSdkHttpClientBuilder; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.http.ExecutableHttpRequest; import software.amazon.awssdk.http.HttpExecuteRequest; import software.amazon.awssdk.http.SdkHttpClient; @@ -205,7 +205,7 @@ private SdkClientConfiguration mergeGlobalDefaults(SdkClientConfiguration config .option(ADDITIONAL_HTTP_HEADERS, new LinkedHashMap<>()) .option(PROFILE_FILE, profileFile) .option(PROFILE_NAME, ProfileFileSystemSetting.AWS_PROFILE.getStringValueOrThrow()) - .option(USER_AGENT_PREFIX, UserAgentUtils.getUserAgent()) + .option(USER_AGENT_PREFIX, SdkUserAgent.create().userAgent()) .option(USER_AGENT_SUFFIX, "") .option(CRC32_FROM_COMPRESSED_DATA_ENABLED, false)); } @@ -298,6 +298,9 @@ protected AttributeMap childHttpConfig() { * Finalize which async executor service will be used for the created client. The default async executor * service has at least 8 core threads and can scale up to at least 64 threads when needed depending * on the number of processors available. + * + * This uses the same default executor in S3NativeClientConfiguration#resolveAsyncFutureCompletionExecutor. + * Make sure you update that method if you update the defaults here. */ private Executor resolveAsyncFutureCompletionExecutor(SdkClientConfiguration config) { Supplier defaultExecutor = () -> { diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncRequestBody.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncRequestBody.java index a0e7c971f41f..4dc230b54c47 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncRequestBody.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/async/FileAsyncRequestBody.java @@ -15,6 +15,8 @@ package software.amazon.awssdk.core.internal.async; +import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; + import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; @@ -31,6 +33,8 @@ import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.internal.util.Mimetype; import software.amazon.awssdk.core.internal.util.NoopSubscription; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.SdkBuilder; /** @@ -41,6 +45,7 @@ */ @SdkInternalApi public final class FileAsyncRequestBody implements AsyncRequestBody { + private static final Logger log = Logger.loggerFor(FileAsyncRequestBody.class); /** * Default size (in bytes) of ByteBuffer chunks read from the file and delivered to the subscriber. @@ -78,17 +83,23 @@ public String contentType() { @Override public void subscribe(Subscriber s) { + AsynchronousFileChannel channel = null; try { - AsynchronousFileChannel channel = openInputChannel(this.path); + channel = openInputChannel(this.path); // We need to synchronize here because the subscriber could call // request() from within onSubscribe which would potentially // trigger onNext before onSubscribe is finished. - Subscription subscription = new FileSubscription(channel, s, chunkSizeInBytes); + // + // Note: size() can throw IOE here + Subscription subscription = new FileSubscription(channel, channel.size(), s, chunkSizeInBytes); synchronized (subscription) { s.onSubscribe(subscription); } } catch (IOException e) { + if (channel != null) { + runAndLogError(log.logger(), "Unable to close file channel", channel::close); + } // subscribe() must return normally, so we need to signal the // failure to open via onError() once onSubscribe() is signaled. s.onSubscribe(new NoopSubscription(s)); @@ -169,15 +180,21 @@ private static final class FileSubscription implements Subscription { private final Subscriber subscriber; private final int chunkSize; - private long position = 0; - private AtomicLong outstandingDemand = new AtomicLong(0); - private boolean writeInProgress = false; + private final AtomicLong position = new AtomicLong(0); + private final AtomicLong remainingBytes = new AtomicLong(0); + private long outstandingDemand = 0; + private boolean readInProgress = false; private volatile boolean done = false; + private final Object lock = new Object(); - private FileSubscription(AsynchronousFileChannel inputChannel, Subscriber subscriber, int chunkSize) { + private FileSubscription(AsynchronousFileChannel inputChannel, + long size, + Subscriber subscriber, + int chunkSize) { this.inputChannel = inputChannel; this.subscriber = subscriber; this.chunkSize = chunkSize; + this.remainingBytes.set(Validate.isNotNegative(size, "size")); } @Override @@ -189,23 +206,24 @@ public void request(long n) { if (n < 1) { IllegalArgumentException ex = new IllegalArgumentException(subscriber + " violated the Reactive Streams rule 3.9 by requesting a " - + "non-positive number of elements."); + + "non-positive number of elements."); signalOnError(ex); } else { try { - // As governed by rule 3.17, when demand overflows `Long.MAX_VALUE` we treat the signalled demand as - // "effectively unbounded" - outstandingDemand.getAndUpdate(initialDemand -> { - if (Long.MAX_VALUE - initialDemand < n) { - return Long.MAX_VALUE; + // We need to synchronize here because of the race condition + // where readData finishes reading at the same time request + // demand comes in + synchronized (lock) { + // As governed by rule 3.17, when demand overflows `Long.MAX_VALUE` we treat the signalled demand as + // "effectively unbounded" + if (Long.MAX_VALUE - outstandingDemand < n) { + outstandingDemand = Long.MAX_VALUE; } else { - return initialDemand + n; + outstandingDemand += n; } - }); - synchronized (this) { - if (!writeInProgress) { - writeInProgress = true; + if (!readInProgress) { + readInProgress = true; readData(); } } @@ -227,31 +245,40 @@ public void cancel() { private void readData() { // It's possible to have another request for data come in after we've closed the file. - if (!inputChannel.isOpen()) { + if (!inputChannel.isOpen() || done) { return; } ByteBuffer buffer = ByteBuffer.allocate(chunkSize); - inputChannel.read(buffer, position, buffer, new CompletionHandler() { + inputChannel.read(buffer, position.get(), buffer, new CompletionHandler() { @Override public void completed(Integer result, ByteBuffer attachment) { if (result > 0) { attachment.flip(); - position += attachment.remaining(); + + int readBytes = attachment.remaining(); + position.addAndGet(readBytes); + remainingBytes.addAndGet(-readBytes); + signalOnNext(attachment); - // If we have more permits, queue up another read. - if (outstandingDemand.decrementAndGet() > 0) { - readData(); - return; + + if (remainingBytes.get() == 0) { + closeFile(); + signalOnComplete(); + } + + synchronized (lock) { + // If we have more permits, queue up another read. + if (--outstandingDemand > 0) { + readData(); + } else { + readInProgress = false; + } } } else { // Reached the end of the file, notify the subscriber and cleanup - signalOnComplete(); closeFile(); - } - - synchronized (FileSubscription.this) { - writeInProgress = false; + signalOnComplete(); } } @@ -267,14 +294,14 @@ private void closeFile() { try { inputChannel.close(); } catch (IOException e) { - signalOnError(e); + log.warn(() -> "Failed to close the file", e); } } - private void signalOnNext(ByteBuffer bb) { + private void signalOnNext(ByteBuffer attachment) { synchronized (this) { if (!done) { - subscriber.onNext(bb); + subscriber.onNext(attachment); } } } @@ -282,8 +309,8 @@ private void signalOnNext(ByteBuffer bb) { private void signalOnComplete() { synchronized (this) { if (!done) { - subscriber.onComplete(); done = true; + subscriber.onComplete(); } } } @@ -291,8 +318,8 @@ private void signalOnComplete() { private void signalOnError(Throwable t) { synchronized (this) { if (!done) { - subscriber.onError(t); done = true; + subscriber.onError(t); } } } @@ -301,4 +328,4 @@ private void signalOnError(Throwable t) { private static AsynchronousFileChannel openInputChannel(Path path) throws IOException { return AsynchronousFileChannel.open(path, StandardOpenOption.READ); } -} +} \ No newline at end of file diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ApplyUserAgentStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ApplyUserAgentStage.java index 948a81529153..d96f1b1b5f39 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ApplyUserAgentStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/ApplyUserAgentStage.java @@ -29,7 +29,7 @@ import software.amazon.awssdk.core.internal.http.HttpClientDependencies; import software.amazon.awssdk.core.internal.http.RequestExecutionContext; import software.amazon.awssdk.core.internal.http.pipeline.MutableRequestToRequestPipeline; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.utils.StringUtils; import software.amazon.awssdk.utils.http.SdkHttpUtils; @@ -71,7 +71,7 @@ private StringBuilder getUserAgent(SdkClientConfiguration config, List StringBuilder userAgent = new StringBuilder(StringUtils.trimToEmpty(userDefinedPrefix)); - String systemUserAgent = UserAgentUtils.getUserAgent(); + String systemUserAgent = SdkUserAgent.create().userAgent(); if (!systemUserAgent.equals(userDefinedPrefix)) { userAgent.append(COMMA).append(systemUserAgent); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UserAgentUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/util/SdkUserAgent.java similarity index 88% rename from core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UserAgentUtils.java rename to core/sdk-core/src/main/java/software/amazon/awssdk/core/util/SdkUserAgent.java index 30f3e6b7022a..f154d7377a26 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/util/UserAgentUtils.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/util/SdkUserAgent.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.internal.util; +package software.amazon.awssdk.core.util; import java.util.Optional; import java.util.jar.JarInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkProtectedApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; import software.amazon.awssdk.annotations.ThreadSafe; -import software.amazon.awssdk.core.util.VersionInfo; import software.amazon.awssdk.utils.IoUtils; import software.amazon.awssdk.utils.JavaSystemSetting; @@ -29,8 +29,8 @@ * Utility class for accessing AWS SDK versioning information. */ @ThreadSafe -@SdkInternalApi -public final class UserAgentUtils { +@SdkProtectedApi +public final class SdkUserAgent { private static final String UA_STRING = "aws-sdk-{platform}/{version} {os.name}/{os.version} {java.vm.name}/{java.vm" + ".version} Java/{java.version}{language.and.region}{additional.languages} " @@ -40,12 +40,27 @@ public final class UserAgentUtils { private static final String UA_BLACKLIST_REGEX = "[() ,/:;<=>?@\\[\\]{}\\\\]"; /** Shared logger for any issues while loading version information. */ - private static final Logger log = LoggerFactory.getLogger(UserAgentUtils.class); + private static final Logger log = LoggerFactory.getLogger(SdkUserAgent.class); private static final String UNKNOWN = "unknown"; + private static volatile SdkUserAgent instance; + /** User Agent info. */ - private static volatile String userAgent; + private String userAgent; + + private SdkUserAgent() { + initializeUserAgent(); + } + + public static SdkUserAgent create() { + if (instance == null) { + synchronized (SdkUserAgent.class) { + if (instance == null) { + instance = new SdkUserAgent(); + } + } + } - private UserAgentUtils() { + return instance; } /** @@ -53,14 +68,7 @@ private UserAgentUtils() { * the AWS services. The User Agent encapsulates SDK, Java, OS and * region information. */ - public static String getUserAgent() { - if (userAgent == null) { - synchronized (UserAgentUtils.class) { - if (userAgent == null) { - initializeUserAgent(); - } - } - } + public String userAgent() { return userAgent; } @@ -69,11 +77,12 @@ public static String getUserAgent() { * {@code InternalConfig} and filling in the detected version/platform * info. */ - private static void initializeUserAgent() { - userAgent = userAgent(); + private void initializeUserAgent() { + userAgent = getUserAgent(); } - static String userAgent() { + @SdkTestInternalApi + String getUserAgent() { String ua = UA_STRING; diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncRequestBodyTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncRequestBodyTest.java new file mode 100644 index 000000000000..c633d6f1d43e --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncRequestBodyTest.java @@ -0,0 +1,95 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.async; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.AsynchronousFileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.testutils.RandomTempFile; + +public class FileAsyncRequestBodyTest { + private static final long MiB = 1024 * 1024; + private static final long TEST_FILE_SIZE = 10 * MiB; + private static Path testFile; + + @BeforeClass + public static void setup() throws IOException { + testFile = new RandomTempFile(TEST_FILE_SIZE).toPath(); + } + + @AfterClass + public static void teardown() throws IOException { + Files.delete(testFile); + } + + // If we issue just enough requests to read the file entirely but not more (to go past EOF), we should still receive + // an onComplete + @Test + public void readFully_doesNotRequestPastEndOfFile_receivesComplete() throws InterruptedException, ExecutionException, TimeoutException { + int chunkSize = 16384; + AsyncRequestBody asyncRequestBody = FileAsyncRequestBody.builder() + .path(testFile) + .chunkSizeInBytes(chunkSize) + .build(); + + long totalRequests = TEST_FILE_SIZE / chunkSize; + + CompletableFuture completed = new CompletableFuture<>(); + asyncRequestBody.subscribe(new Subscriber() { + private Subscription sub; + private long requests = 0; + @Override + public void onSubscribe(Subscription subscription) { + this.sub = subscription; + if (requests++ < totalRequests) { + this.sub.request(1); + } + } + + @Override + public void onNext(ByteBuffer byteBuffer) { + if (requests++ < totalRequests) { + this.sub.request(1); + } + } + + @Override + public void onError(Throwable throwable) { + } + + @Override + public void onComplete() { + completed.complete(null); + } + }); + + completed.get(5, TimeUnit.SECONDS); + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java index 015ecbdcca9e..47684bd4b280 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/async/FileAsyncResponseTransformerTest.java @@ -128,4 +128,4 @@ public void cancel() { }); } } -} +} \ No newline at end of file diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/UserAgentUtilsTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/util/SdkUserAgentTest.java similarity index 83% rename from core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/UserAgentUtilsTest.java rename to core/sdk-core/src/test/java/software/amazon/awssdk/core/util/SdkUserAgentTest.java index 693e4c2168d4..4d4430833fa2 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/util/UserAgentUtilsTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/util/SdkUserAgentTest.java @@ -13,21 +13,21 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.core.internal.util; +package software.amazon.awssdk.core.util; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertNotNull; import java.util.Arrays; import org.junit.Test; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.utils.JavaSystemSetting; -public class UserAgentUtilsTest { +public class SdkUserAgentTest { @Test public void userAgent() { - String userAgent = UserAgentUtils.userAgent(); + String userAgent = SdkUserAgent.create().userAgent(); assertNotNull(userAgent); Arrays.stream(userAgent.split(" ")).forEach(str -> assertThat(isValidInput(str)).isTrue()); } @@ -35,7 +35,7 @@ public void userAgent() { @Test public void userAgent_HasVendor() { System.setProperty(JavaSystemSetting.JAVA_VENDOR.property(), "finks"); - String userAgent = UserAgentUtils.userAgent(); + String userAgent = SdkUserAgent.create().getUserAgent(); System.clearProperty(JavaSystemSetting.JAVA_VENDOR.property()); assertThat(userAgent).contains("vendor/finks"); } @@ -43,7 +43,7 @@ public void userAgent_HasVendor() { @Test public void userAgent_HasUnknownVendor() { System.clearProperty(JavaSystemSetting.JAVA_VENDOR.property()); - String userAgent = UserAgentUtils.userAgent(); + String userAgent = SdkUserAgent.create().getUserAgent(); assertThat(userAgent).contains("vendor/unknown"); } diff --git a/pom.xml b/pom.xml index 236b5e31bb29..f8a7d441da5a 100644 --- a/pom.xml +++ b/pom.xml @@ -80,6 +80,7 @@ test/tests-coverage-reporting test/stability-tests test/sdk-native-image-test + test/s3-benchmarks test/auth-sts-testing @@ -112,7 +113,7 @@ 2.2.21 1.10 1.29 - 0.13.11 + 0.13.14 4.13.1 @@ -479,6 +480,14 @@ + + + software/amazon/awssdk/modulepath/tests/**/*.class + software/amazon/awssdk/nativeimagetest/**/*.class + software/amazon/awssdk/testutils/service/**/*.class + **/*Benchmark.class + + @@ -530,8 +539,10 @@ archetype-tools sdk-benchmarks bundle + s3-benchmarks aws-crt-client + s3-transfer-manager true true diff --git a/services-custom/pom.xml b/services-custom/pom.xml index b5bce56d6c27..c2ad9e6fc71e 100644 --- a/services-custom/pom.xml +++ b/services-custom/pom.xml @@ -29,6 +29,7 @@ dynamodb-enhanced + s3-transfer-manager diff --git a/services-custom/s3-transfer-manager/README.md b/services-custom/s3-transfer-manager/README.md new file mode 100644 index 000000000000..54f78e8d3925 --- /dev/null +++ b/services-custom/s3-transfer-manager/README.md @@ -0,0 +1,65 @@ +## Overview + +This project provides a much improved experience for S3 customers needing to easily perform uploads and downloads of +objects to and from S3 by providing the S3 S3TransferManager, a high level +library built on the [AWS Common Runtime S3 Client](https://github.com/awslabs/aws-crt-java). + +## Getting Started + +### Add a dependency for the transfer manager + +First, you need to include the dependency in your project. + +```xml + + software.amazon.awssdk + s3-transfer-manager + ${awsjavasdk.version}-PREVIEW + +``` + +Note that you need to replace `${awsjavasdk.version}` with the latest +SDK version + +### Instantiate the transfer manager +You can instantiate the transfer manager easily using the default settings + +```java + +S3TransferManager transferManager = S3TransferManager.create(); + +``` + +If you wish to configure settings, we recommend using the builder instead: +```java +S3TransferManager transferManager = + S3TransferManager.builder() + .s3ClientConfiguration(b -> b.credentialsProvider(credentialProvider) + .region(Region.US_WEST_2) + .targetThroughputInGbps(20.0) + .minimumPartSizeInBytes(10 * MB)) + .build(); +``` + +### Download an S3 object to a file +To download an object, you just need to provide the destion file path and the `GetObjectRequest` that should be used for the download. + +```java +Download download = + transferManager.download(b -> b.destination(path) + .getObjectRequest(r -> r.bucket("bucket") + .key("key"))); +download.completionFuture().join(); +``` + +### Upload a file to S3 +To upload a file to S3, you just need to provide the source file path and the `PutObjectRequest` that should be used for the upload. + +```java +Upload upload = transferManager.upload(b -> b.source(path) + .putObjectRequest(r -> r.bucket("bucket") + .key("key"))); + +upload.completionFuture().join(); + +``` \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/pom.xml b/services-custom/s3-transfer-manager/pom.xml new file mode 100644 index 000000000000..6dd6ed613c40 --- /dev/null +++ b/services-custom/s3-transfer-manager/pom.xml @@ -0,0 +1,174 @@ + + + + + 4.0.0 + + software.amazon.awssdk + aws-sdk-java-pom + 2.17.16-SNAPSHOT + ../../pom.xml + + s3-transfer-manager + ${awsjavasdk.version}-PREVIEW + AWS Java SDK :: S3 :: Transfer Manager + + The S3 Transfer Manager allows customers to easily and optimally + transfer objects and directories to and from S3. + + https://aws.amazon.com/sdkforjava + + + 1.8 + ${project.parent.version} + + + + + + software.amazon.awssdk + bom-internal + ${awsjavasdk.version} + pom + import + + + + + + + software.amazon.awssdk + s3 + ${awsjavasdk.version} + + + software.amazon.awssdk + sdk-core + ${awsjavasdk.version} + + + software.amazon.awssdk + utils + ${awsjavasdk.version} + + + software.amazon.awssdk + annotations + ${awsjavasdk.version} + + + software.amazon.awssdk + regions + ${awsjavasdk.version} + + + software.amazon.awssdk.crt + aws-crt + ${awscrt.version} + + + software.amazon.awssdk + http-client-spi + ${awsjavasdk.version} + + + software.amazon.awssdk + auth + ${awsjavasdk.version} + + + software.amazon.awssdk + aws-core + ${awsjavasdk.version} + + + software.amazon.awssdk + metrics-spi + ${awsjavasdk.version} + + + software.amazon.awssdk + service-test-utils + ${awsjavasdk.version} + test + + + software.amazon.awssdk + test-utils + ${awsjavasdk.version} + test + + + junit + junit + test + + + org.assertj + assertj-core + test + + + org.mockito + mockito-core + test + + + log4j + log4j + test + + + org.slf4j + slf4j-log4j12 + test + + + io.reactivex.rxjava2 + rxjava + test + + + org.apache.commons + commons-lang3 + test + + + org.reactivestreams + reactive-streams-tck + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + software.amazon.awssdk.transfer.s3 + + + + + + + + diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/CrtExceptionTransformationIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/CrtExceptionTransformationIntegrationTest.java new file mode 100644 index 000000000000..f61714e4efc4 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/CrtExceptionTransformationIntegrationTest.java @@ -0,0 +1,137 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +public class CrtExceptionTransformationIntegrationTest extends S3IntegrationTestBase { + + private static final String BUCKET = temporaryBucketName(CrtExceptionTransformationIntegrationTest.class); + + private static final String KEY = "some-key"; + + private static final int OBJ_SIZE = 8 * 1024; + private static RandomTempFile testFile; + private static S3TransferManager transferManager; + private static S3CrtAsyncClient s3Crt; + + @BeforeClass + public static void setupFixture() throws IOException { + createBucket(BUCKET); + testFile = new RandomTempFile(BUCKET, OBJ_SIZE); + s3Crt = S3CrtAsyncClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(S3IntegrationTestBase.DEFAULT_REGION) + .build(); + transferManager = + S3TransferManager.builder() + .s3ClientConfiguration(b -> b.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(S3IntegrationTestBase.DEFAULT_REGION)) + .build(); + } + + @AfterClass + public static void tearDownFixture() { + deleteBucketAndAllContents(BUCKET); + s3Crt.close(); + transferManager.close(); + testFile.delete(); + } + + @Test + public void getObjectNoSuchKey() throws IOException { + String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString(); + assertThatThrownBy(() -> s3Crt.getObject(GetObjectRequest.builder().bucket(BUCKET).key("randomKey").build(), + Paths.get(randomBaseDirectory).resolve("testFile")).get()) + .hasCauseInstanceOf(NoSuchKeyException.class) + .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchKeyException: The specified key does not exist"); + } + + @Test + public void getObjectNoSuchBucket() throws IOException { + String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString(); + assertThatThrownBy(() -> s3Crt.getObject(GetObjectRequest.builder().bucket("nonExistingTestBucket").key(KEY).build(), + Paths.get(randomBaseDirectory).resolve("testFile")).get()) + .hasCauseInstanceOf(NoSuchBucketException.class) + .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist"); + } + + @Test + public void transferManagerDownloadObjectWithNoSuchKey() throws IOException { + String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString(); + assertThatThrownBy(() -> transferManager.download(DownloadRequest.builder() + .getObjectRequest(GetObjectRequest.builder().bucket(BUCKET).key("randomKey").build()) + .destination(Paths.get(randomBaseDirectory).resolve("testFile")) + .build()).completionFuture().join()) + .hasCauseInstanceOf(NoSuchKeyException.class) + .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchKeyException: The specified key does not exist"); + } + + @Test + public void transferManagerDownloadObjectWithNoSuchBucket() throws IOException { + String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString(); + assertThatThrownBy(() -> transferManager.download(DownloadRequest.builder() + .getObjectRequest(GetObjectRequest.builder().bucket("nonExistingTestBucket").key(KEY).build()) + .destination(Paths.get(randomBaseDirectory).resolve("testFile")) + .build()).completionFuture().join()) + .hasCauseInstanceOf(NoSuchBucketException.class) + .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist"); + } + + @Test + public void putObjectNoSuchKey() throws IOException { + String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString(); + assertThatThrownBy(() -> s3Crt.getObject(GetObjectRequest.builder().bucket(BUCKET).key("someRandomKey").build(), + Paths.get(randomBaseDirectory).resolve("testFile")).get()) + .hasCauseInstanceOf(NoSuchKeyException.class) + .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchKeyException: The specified key does not exist"); + } + + @Test + public void putObjectNoSuchBucket() throws IOException { + + String randomBaseDirectory = Files.createTempDirectory(getClass().getSimpleName()).toString(); + assertThatThrownBy(() -> s3Crt.getObject(GetObjectRequest.builder().bucket("nonExistingTestBucket").key(KEY).build(), + Paths.get(randomBaseDirectory).resolve("testFile")).get()) + .hasCauseInstanceOf(NoSuchBucketException.class) + .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist"); + } + + @Test + public void transferManagerUploadObjectWithNoSuchObject() throws IOException{ + assertThatThrownBy(() -> transferManager.upload(UploadRequest.builder() + .putObjectRequest(PutObjectRequest.builder().bucket("nonExistingTestBucket").key("someKey").build()) + .source(testFile.toPath()) + .build()).completionFuture().join()) + .hasCauseInstanceOf(NoSuchBucketException.class) + .hasMessageContaining("software.amazon.awssdk.services.s3.model.NoSuchBucketException: The specified bucket does not exist"); + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtClientPutObjectIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtClientPutObjectIntegrationTest.java new file mode 100644 index 000000000000..9e5fa71bb0b5 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtClientPutObjectIntegrationTest.java @@ -0,0 +1,179 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import io.reactivex.Flowable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.assertj.core.api.Assertions; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.reactivestreams.Subscriber; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient; +import software.amazon.awssdk.transfer.s3.util.ChecksumUtils; + +public class S3CrtClientPutObjectIntegrationTest extends S3IntegrationTestBase { + private static final String TEST_BUCKET = temporaryBucketName(S3CrtClientPutObjectIntegrationTest.class); + private static final String TEST_KEY = "8mib_file.dat"; + private static final int OBJ_SIZE = 8 * 1024 * 1024; + + private static RandomTempFile testFile; + private static ExecutorService executorService; + private S3CrtAsyncClient s3Crt; + + + @BeforeClass + public static void setup() throws Exception { + S3IntegrationTestBase.setUp(); + S3IntegrationTestBase.createBucket(TEST_BUCKET); + + testFile = new RandomTempFile(TEST_KEY, OBJ_SIZE); + executorService = Executors.newFixedThreadPool(2); + } + + @Before + public void methodSetup() { + s3Crt = S3CrtAsyncClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(S3IntegrationTestBase.DEFAULT_REGION) + .build(); + } + + @After + public void methodTeardown() { + s3Crt.close(); + } + + @AfterClass + public static void teardown() throws IOException { + S3IntegrationTestBase.deleteBucketAndAllContents(TEST_BUCKET); + Files.delete(testFile.toPath()); + executorService.shutdown(); + S3IntegrationTestBase.cleanUp(); + } + + @Test + public void putObject_fileRequestBody_objectSentCorrectly() throws IOException, NoSuchAlgorithmException { + AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath()); + s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); + + ResponseInputStream objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); + + byte[] expectedSum = ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath())); + + Assertions.assertThat(ChecksumUtils.computeCheckSum(objContent)).isEqualTo(expectedSum); + } + + @Test + public void putObject_byteBufferBody_objectSentCorrectly() throws IOException, NoSuchAlgorithmException { + byte[] data = new byte[16384]; + new Random().nextBytes(data); + ByteBuffer byteBuffer = ByteBuffer.wrap(data); + + AsyncRequestBody body = AsyncRequestBody.fromByteBuffer(byteBuffer); + + s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); + + ResponseBytes responseBytes = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toBytes()); + + byte[] expectedSum = ChecksumUtils.computeCheckSum(byteBuffer); + + Assertions.assertThat(ChecksumUtils.computeCheckSum(responseBytes.asByteBuffer())).isEqualTo(expectedSum); + } + + @Test + public void putObject_customRequestBody_objectSentCorrectly() throws IOException, NoSuchAlgorithmException { + Random rng = new Random(); + int bufferSize = 16384; + int nBuffers = 15; + List bodyData = Stream.generate(() -> { + byte[] data = new byte[bufferSize]; + rng.nextBytes(data); + return ByteBuffer.wrap(data); + }).limit(nBuffers).collect(Collectors.toList()); + + long contentLength = bufferSize * nBuffers; + + byte[] expectedSum = ChecksumUtils.computeCheckSum(bodyData); + + Flowable publisher = Flowable.fromIterable(bodyData); + + AsyncRequestBody customRequestBody = new AsyncRequestBody() { + @Override + public Optional contentLength() { + return Optional.of(contentLength); + } + + @Override + public void subscribe(Subscriber subscriber) { + publisher.subscribe(subscriber); + } + }; + + s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), customRequestBody).join(); + + ResponseInputStream objContent = S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); + + + Assertions.assertThat(ChecksumUtils.computeCheckSum(objContent)).isEqualTo(expectedSum); + } + + @Test + public void putObject_customExecutorService_objectSentCorrectly() throws IOException { + AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath()); + try (S3CrtAsyncClient s3Client = + S3CrtAsyncClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(DEFAULT_REGION) + .asyncConfiguration(b -> b.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, + executorService)) + .build()) { + + s3Client.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); + ResponseInputStream objContent = + S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); + + byte[] expectedSum = ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath())); + + Assertions.assertThat(ChecksumUtils.computeCheckSum(objContent)).isEqualTo(expectedSum); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtGetObjectIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtGetObjectIntegrationTest.java new file mode 100644 index 000000000000..eb80d2ae9058 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtGetObjectIntegrationTest.java @@ -0,0 +1,146 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.http.async.SimpleSubscriber; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.testutils.service.AwsTestBase; +import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient; +import software.amazon.awssdk.utils.Md5Utils; + +public class S3CrtGetObjectIntegrationTest extends S3IntegrationTestBase { + private static final String BUCKET = temporaryBucketName(S3CrtGetObjectIntegrationTest.class); + private static final String KEY = "key"; + private static S3CrtAsyncClient crtClient; + private static File file; + private static ExecutorService executorService; + + @BeforeClass + public static void setup() throws IOException { + S3IntegrationTestBase.createBucket(BUCKET); + crtClient = S3CrtAsyncClient.builder() + .region(S3IntegrationTestBase.DEFAULT_REGION) + .credentialsProvider(AwsTestBase.CREDENTIALS_PROVIDER_CHAIN) + .build(); + file = new RandomTempFile(10_000); + S3IntegrationTestBase.s3.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(KEY) + .build(), file.toPath()); + executorService = Executors.newFixedThreadPool(2); + } + + @AfterClass + public static void cleanup() { + crtClient.close(); + S3IntegrationTestBase.deleteBucketAndAllContents(BUCKET); + executorService.shutdown(); + S3IntegrationTestBase.cleanUp(); + } + + @Test + public void getObject_toFiles() throws IOException { + Path path = RandomTempFile.randomUncreatedFile().toPath(); + + GetObjectResponse response = + crtClient.getObject(b -> b.bucket(BUCKET).key(KEY), AsyncResponseTransformer.toFile(path)).join(); + + assertThat(Md5Utils.md5AsBase64(path.toFile())).isEqualTo(Md5Utils.md5AsBase64(file)); + } + + @Test + public void getObject_toBytes() throws IOException { + byte[] bytes = + crtClient.getObject(b -> b.bucket(BUCKET).key(KEY), AsyncResponseTransformer.toBytes()).join().asByteArray(); + assertThat(bytes).isEqualTo(Files.readAllBytes(file.toPath())); + } + + @Test + public void getObject_customResponseTransformer() { + crtClient.getObject(b -> b.bucket(BUCKET).key(KEY), + new TestResponseTransformer()).join(); + + } + + @Test + public void getObject_customExecutors_fileDownloadCorrectly() throws IOException { + Path path = RandomTempFile.randomUncreatedFile().toPath(); + + try (S3CrtAsyncClient s3Client = + S3CrtAsyncClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(DEFAULT_REGION) + .asyncConfiguration(b -> b.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, + executorService)) + .build()) { + GetObjectResponse response = + s3Client.getObject(b -> b.bucket(BUCKET).key(KEY), AsyncResponseTransformer.toFile(path)).join(); + + assertThat(Md5Utils.md5AsBase64(path.toFile())).isEqualTo(Md5Utils.md5AsBase64(file)); + } + } + + private static final class TestResponseTransformer implements AsyncResponseTransformer { + private CompletableFuture future; + + @Override + public CompletableFuture prepare() { + future = new CompletableFuture<>(); + return future; + } + + @Override + public void onResponse(GetObjectResponse response) { + assertThat(response).isNotNull(); + } + + @Override + public void onStream(SdkPublisher publisher) { + publisher.subscribe(new SimpleSubscriber(b -> { + }) { + @Override + public void onComplete() { + super.onComplete(); + future.complete(null); + } + }); + } + + @Override + public void exceptionOccurred(Throwable error) { + future.completeExceptionally(error); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java new file mode 100644 index 000000000000..2dcd74f9bd5a --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java @@ -0,0 +1,149 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.BucketLocationConstraint; +import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectVersion; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.testutils.service.AwsTestBase; + +/** + * Base class for S3 integration tests. Loads AWS credentials from a properties + * file and creates an S3 client for callers to use. + */ +public class S3IntegrationTestBase extends AwsTestBase { + + protected static final Region DEFAULT_REGION = Region.US_WEST_2; + /** + * The S3 client for all tests to use. + */ + protected static S3Client s3; + + protected static S3AsyncClient s3Async; + + /** + * Loads the AWS account info for the integration tests and creates an S3 + * client for tests to use. + */ + @BeforeClass + public static void setUp() throws Exception { + System.setProperty("aws.crt.debugnative", "true"); + s3 = s3ClientBuilder().build(); + s3Async = s3AsyncClientBuilder().build(); + } + + @AfterClass + public static void cleanUp() { + CrtResource.waitForNoResources(); + } + + protected static S3ClientBuilder s3ClientBuilder() { + return S3Client.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN); + } + + protected static S3AsyncClientBuilder s3AsyncClientBuilder() { + return S3AsyncClient.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN); + } + + protected static void createBucket(String bucketName) { + createBucket(bucketName, 0); + } + + private static void createBucket(String bucketName, int retryCount) { + try { + s3.createBucket( + CreateBucketRequest.builder() + .bucket(bucketName) + .createBucketConfiguration( + CreateBucketConfiguration.builder() + .locationConstraint(BucketLocationConstraint.US_WEST_2) + .build()) + .build()); + } catch (S3Exception e) { + System.err.println("Error attempting to create bucket: " + bucketName); + if (e.awsErrorDetails().errorCode().equals("BucketAlreadyOwnedByYou")) { + System.err.printf("%s bucket already exists, likely leaked by a previous run\n", bucketName); + } else if (e.awsErrorDetails().errorCode().equals("TooManyBuckets")) { + System.err.println("Printing all buckets for debug:"); + s3.listBuckets().buckets().forEach(System.err::println); + if (retryCount < 2) { + System.err.println("Retrying..."); + createBucket(bucketName, retryCount + 1); + } else { + throw e; + } + } else { + throw e; + } + } + } + + protected static void deleteBucketAndAllContents(String bucketName) { + System.out.println("Deleting S3 bucket: " + bucketName); + ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build()); + + while (true) { + if (response.contents() == null) { + break; + } + for (S3Object objectSummary : response.contents()) { + s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build()); + } + + if (response.isTruncated()) { + response = s3.listObjects(ListObjectsRequest.builder().marker(response.nextMarker()).build()); + } else { + break; + } + } + + ListObjectVersionsResponse versionsResponse = s3 + .listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build()); + if (versionsResponse.versions() != null) { + for (ObjectVersion s : versionsResponse.versions()) { + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(s.key()) + .versionId(s.versionId()) + .build()); + } + } + + s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()); + } + +} diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java new file mode 100644 index 000000000000..52e49e587807 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadIntegrationTest.java @@ -0,0 +1,67 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.utils.Md5Utils; + +public class S3TransferManagerDownloadIntegrationTest extends S3IntegrationTestBase { + private static final String BUCKET = temporaryBucketName(S3TransferManagerDownloadIntegrationTest.class); + private static final String KEY = "key"; + private static S3TransferManager transferManager; + private static File file; + + @BeforeClass + public static void setup() throws IOException { + createBucket(BUCKET); + file = new RandomTempFile(10_000); + s3.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(KEY) + .build(), file.toPath()); + transferManager = S3TransferManager.builder() + .s3ClientConfiguration(b -> b.region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)) + .build(); + } + + @AfterClass + public static void cleanup() { + deleteBucketAndAllContents(BUCKET); + transferManager.close(); + S3IntegrationTestBase.cleanUp(); + } + + @Test + public void download_shouldWork() throws IOException { + Path path = RandomTempFile.randomUncreatedFile().toPath(); + Download download = transferManager.download(b -> b.getObjectRequest(r -> r.bucket(BUCKET).key(KEY)) + .destination(path)); + CompletedDownload completedDownload = download.completionFuture().join(); + assertThat(Md5Utils.md5AsBase64(path.toFile())).isEqualTo(Md5Utils.md5AsBase64(file)); + assertThat(completedDownload.response().responseMetadata().requestId()).isNotNull(); + } +} diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java new file mode 100644 index 000000000000..bd006f9511e5 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerUploadIntegrationTest.java @@ -0,0 +1,80 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; +import java.io.IOException; +import java.nio.file.Files; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.transfer.s3.util.ChecksumUtils; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.testutils.RandomTempFile; + +public class S3TransferManagerUploadIntegrationTest extends S3IntegrationTestBase { + private static final String TEST_BUCKET = temporaryBucketName(S3TransferManagerUploadIntegrationTest.class); + private static final String TEST_KEY = "8mib_file.dat"; + private static final int OBJ_SIZE = 8 * 1024 * 1024; + + private static RandomTempFile testFile; + private static S3TransferManager tm; + + @BeforeClass + public static void setUp() throws Exception { + S3IntegrationTestBase.setUp(); + createBucket(TEST_BUCKET); + + testFile = new RandomTempFile(TEST_KEY, OBJ_SIZE); + + tm = S3TransferManager.builder() + .s3ClientConfiguration(b -> b.credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(DEFAULT_REGION) + .maxConcurrency(100)) + .build(); + + } + + @AfterClass + public static void teardown() throws IOException { + tm.close(); + Files.delete(testFile.toPath()); + deleteBucketAndAllContents(TEST_BUCKET); + S3IntegrationTestBase.cleanUp(); + } + + @Test + public void upload_fileSentCorrectly() throws IOException { + Upload upload = tm.upload(UploadRequest.builder() + .putObjectRequest(b -> b.bucket(TEST_BUCKET).key(TEST_KEY)) + .source(testFile.toPath()) + .build()); + + CompletedUpload completedUpload = upload.completionFuture().join(); + assertThat(completedUpload.response().responseMetadata().requestId()).isNotNull(); + assertThat(completedUpload.response().sdkHttpResponse()).isNotNull(); + + ResponseInputStream obj = s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), + ResponseTransformer.toInputStream()); + + assertThat(ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath()))) + .isEqualTo(ChecksumUtils.computeCheckSum(obj)); + assertThat(obj.response().responseMetadata().requestId()).isNotNull(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDownload.java new file mode 100644 index 000000000000..1fb176f93cd3 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedDownload.java @@ -0,0 +1,34 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +/** + * A completed download transfer. + */ +@SdkPublicApi +@SdkPreviewApi +public interface CompletedDownload extends CompletedTransfer { + + /** + * Returns the API response from the {@link S3TransferManager#download(DownloadRequest)} + * @return the response + */ + GetObjectResponse response(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedTransfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedTransfer.java new file mode 100644 index 000000000000..28ccbf0940ba --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedTransfer.java @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A completed transfer. + */ +@SdkPublicApi +@SdkPreviewApi +public interface CompletedTransfer { +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java new file mode 100644 index 000000000000..5633084018a0 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/CompletedUpload.java @@ -0,0 +1,34 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +/** + * A completed upload transfer. + */ +@SdkPublicApi +@SdkPreviewApi +public interface CompletedUpload extends CompletedTransfer { + + /** + * Returns the API response from the {@link S3TransferManager#upload(UploadRequest)} + * @return the response + */ + PutObjectResponse response(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Download.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Download.java new file mode 100644 index 000000000000..69319fd196b6 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Download.java @@ -0,0 +1,31 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * A download transfer of a single object from S3. + */ +@SdkPublicApi +@SdkPreviewApi +public interface Download extends Transfer { + + @Override + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadRequest.java new file mode 100644 index 000000000000..f2191c905660 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadRequest.java @@ -0,0 +1,173 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.nio.file.Path; +import java.util.Objects; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.NotThreadSafe; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Request object to download an object from S3 using the Transfer Manager. + */ +@SdkPublicApi +@SdkPreviewApi +public final class DownloadRequest implements TransferRequest, ToCopyableBuilder { + private final Path destination; + private final GetObjectRequest getObjectRequest; + + private DownloadRequest(BuilderImpl builder) { + this.destination = Validate.paramNotNull(builder.destination, "destination"); + this.getObjectRequest = Validate.paramNotNull(builder.getObjectRequest, "getObjectRequest"); + } + + /** + * Create a builder that can be used to create a {@link DownloadRequest}. + * + * @see S3TransferManager#download(DownloadRequest) + */ + public static Builder builder() { + return new BuilderImpl(); + } + + @Override + public Builder toBuilder() { + return new BuilderImpl(); + } + + /** + * The {@link Path} to file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @return the destination path + */ + public Path destination() { + return destination; + } + + /** + * @return The {@link GetObjectRequest} request that should be used for the download + */ + public GetObjectRequest getObjectRequest() { + return getObjectRequest; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DownloadRequest that = (DownloadRequest) o; + + if (!Objects.equals(destination, that.destination)) { + return false; + } + return Objects.equals(getObjectRequest, that.getObjectRequest); + } + + @Override + public int hashCode() { + int result = destination != null ? destination.hashCode() : 0; + result = 31 * result + (getObjectRequest != null ? getObjectRequest.hashCode() : 0); + return result; + } + + /** + * A builder for a {@link DownloadRequest}, created with {@link #builder()} + */ + @SdkPublicApi + @NotThreadSafe + public interface Builder extends TransferRequest.Builder, CopyableBuilder { + + /** + * The {@link Path} to file that response contents will be written to. The file must not exist or this method + * will throw an exception. If the file is not writable by the current user then an exception will be thrown. + * + * @param destination the destination path + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder destination(Path destination); + + /** + * The {@link GetObjectRequest} request that should be used for the download + * + * @param getObjectRequest the getObject request + * @return a reference to this object so that method calls can be chained together. + * @see #getObjectRequest(Consumer) + */ + Builder getObjectRequest(GetObjectRequest getObjectRequest); + + /** + * The {@link GetObjectRequest} request that should be used for the download + * + *

+ * This is a convenience method that creates an instance of the {@link GetObjectRequest} builder avoiding the + * need to create one manually via {@link GetObjectRequest#builder()}. + * + * @param getObjectRequestBuilder the getObject request + * @return a reference to this object so that method calls can be chained together. + * @see #getObjectRequest(GetObjectRequest) + */ + default Builder getObjectRequest(Consumer getObjectRequestBuilder) { + GetObjectRequest request = GetObjectRequest.builder() + .applyMutation(getObjectRequestBuilder) + .build(); + getObjectRequest(request); + return this; + } + + /** + * @return The built request. + */ + DownloadRequest build(); + } + + private static final class BuilderImpl implements Builder { + private Path destination; + private GetObjectRequest getObjectRequest; + + private BuilderImpl() { + } + + @Override + public Builder destination(Path destination) { + this.destination = destination; + return this; + } + + @Override + public Builder getObjectRequest(GetObjectRequest getObjectRequest) { + this.getObjectRequest = getObjectRequest; + return this; + } + + @Override + public DownloadRequest build() { + return new DownloadRequest(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3ClientConfiguration.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3ClientConfiguration.java new file mode 100644 index 000000000000..3acacb7804a3 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3ClientConfiguration.java @@ -0,0 +1,319 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Optional Configurations for the underlying S3 client for which the TransferManager already provides + * sensible defaults. + * + *

Use {@link #builder()} to create a set of options.

+ */ +@SdkPublicApi +@SdkPreviewApi +public final class S3ClientConfiguration implements ToCopyableBuilder { + private final AwsCredentialsProvider credentialsProvider; + private final Region region; + private final Long minimumPartSizeInBytes; + private final Double targetThroughputInGbps; + private final Integer maxConcurrency; + private final ClientAsyncConfiguration asyncConfiguration; + + private S3ClientConfiguration(DefaultBuilder builder) { + this.credentialsProvider = builder.credentialsProvider; + this.region = builder.region; + this.minimumPartSizeInBytes = Validate.isPositiveOrNull(builder.minimumPartSizeInBytes, "minimumPartSizeInBytes"); + this.targetThroughputInGbps = Validate.isPositiveOrNull(builder.targetThroughputInGbps, "targetThroughputInGbps"); + this.maxConcurrency = Validate.isPositiveOrNull(builder.maxConcurrency, + "maxConcurrency"); + this.asyncConfiguration = builder.asyncConfiguration; + } + + /** + * @return the optional credentials that should be used to authenticate with S3. + */ + public Optional credentialsProvider() { + return Optional.ofNullable(credentialsProvider); + } + + /** + * @return the optional region with which the SDK should communicate. + */ + public Optional region() { + return Optional.ofNullable(region); + } + + /** + * @return the optional minimum part size for transfer parts. + */ + public Optional minimumPartSizeInBytes() { + return Optional.ofNullable(minimumPartSizeInBytes); + } + + /** + * @return the optional target throughput + */ + public Optional targetThroughputInGbps() { + return Optional.ofNullable(targetThroughputInGbps); + } + + /** + * @return the optional maximum number of concurrent Amazon S3 transfer requests that can run at the same time. + */ + public Optional maxConcurrency() { + return Optional.ofNullable(maxConcurrency); + } + + /** + * @return the optional SDK async configuration specified + */ + public Optional asyncConfiguration() { + return Optional.ofNullable(asyncConfiguration); + } + + @Override + public Builder toBuilder() { + return new DefaultBuilder(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + S3ClientConfiguration that = (S3ClientConfiguration) o; + + if (!Objects.equals(credentialsProvider, that.credentialsProvider)) { + return false; + } + if (!Objects.equals(region, that.region)) { + return false; + } + if (!Objects.equals(minimumPartSizeInBytes, that.minimumPartSizeInBytes)) { + return false; + } + if (!Objects.equals(targetThroughputInGbps, that.targetThroughputInGbps)) { + return false; + } + if (!Objects.equals(maxConcurrency, that.maxConcurrency)) { + return false; + } + return Objects.equals(asyncConfiguration, that.asyncConfiguration); + } + + @Override + public int hashCode() { + int result = credentialsProvider != null ? credentialsProvider.hashCode() : 0; + result = 31 * result + (region != null ? region.hashCode() : 0); + result = 31 * result + (minimumPartSizeInBytes != null ? minimumPartSizeInBytes.hashCode() : 0); + result = 31 * result + (targetThroughputInGbps != null ? targetThroughputInGbps.hashCode() : 0); + result = 31 * result + (maxConcurrency != null ? maxConcurrency.hashCode() : 0); + result = 31 * result + (asyncConfiguration != null ? asyncConfiguration.hashCode() : 0); + return result; + } + + /** + * Creates a default builder for {@link S3ClientConfiguration}. + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + /** + * The builder definition for a {@link S3ClientConfiguration}. + */ + public interface Builder extends CopyableBuilder { + + /** + * Configure the credentials that should be used to authenticate with S3. + * + *

The default provider will attempt to identify the credentials automatically using the following checks: + *

    + *
  1. Java System Properties - aws.accessKeyId and aws.secretKey
    2. + *
  2. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
    3. + *
  3. Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI
    4. + *
  4. Credentials delivered through the Amazon EC2 container service if AWS_CONTAINER_CREDENTIALS_RELATIVE_URI + * environment variable is set and security manager has permission to access the variable.
    5. + *
  5. Instance profile credentials delivered through the Amazon EC2 metadata service
    6. + *
+ * + *

If the credentials are not found in any of the locations above, an exception will be thrown at {@link #build()} + * time. + *

+ * + * @param credentialsProvider the credentials to use + * @return This builder for method chaining. + */ + Builder credentialsProvider(AwsCredentialsProvider credentialsProvider); + + /** + * Configure the region with which the SDK should communicate. + * + *

If this is not specified, the SDK will attempt to identify the endpoint automatically using the following logic: + *

    + *
  1. Check the 'aws.region' system property for the region.
    2. + *
  2. Check the 'AWS_REGION' environment variable for the region.
    3. + *
  3. Check the {user.home}/.aws/credentials and {user.home}/.aws/config files for the region.
    4. + *
  4. If running in EC2, check the EC2 metadata service for the region.
    5. + *
+ * + * @param region the region to be used + * @return this builder for method chaining. + */ + Builder region(Region region); + + /** + * Sets the minimum part size for transfer parts. Decreasing the minimum part size causes + * multipart transfer to be split into a larger number of smaller parts. Setting this value too low + * has a negative effect on transfer speeds, causing extra latency and network communication for each part. + * + *

+ * By default, it is 8MB + * + * @param partSizeBytes The minimum part size for transfer parts. + * @return this builder for method chaining. + */ + Builder minimumPartSizeInBytes(Long partSizeBytes); + + /** + * The target throughput for transfer requests. Higher value means more S3 connections + * will be opened. Whether the transfer manager can achieve the configured target throughput depends + * on various factors such as the network bandwidth of the environment and the configured {@link #maxConcurrency}. + * + *

+ * By default, it is 5Gbps + * + * @param targetThroughputInGbps the target throughput in Gbps + * @return this builder for method chaining. + * @see #maxConcurrency(Integer) + */ + Builder targetThroughputInGbps(Double targetThroughputInGbps); + + /** + * Specifies the maximum number of S3 connections that should be established during + * a transfer. + * + *

+ * If not provided, the TransferManager will calculate the optional number of connections + * based on {@link #targetThroughputInGbps}. If the value is too low, the S3TransferManager + * might not achieve the specified target throughput. + * + * @param maxConcurrency the max number of concurrent requests + * @return this builder for method chaining. + * @see #targetThroughputInGbps(Double) + */ + Builder maxConcurrency(Integer maxConcurrency); + + /** + * Specify overrides to the default SDK async configuration that should be used for clients created by this builder. + * + * @param asyncConfiguration the async configuration + * @return this builder for method chaining. + * @see #asyncConfiguration(Consumer) + */ + Builder asyncConfiguration(ClientAsyncConfiguration asyncConfiguration); + + /** + * Similar to {@link #asyncConfiguration(ClientAsyncConfiguration)}, but takes a lambda to configure a new + * {@link ClientAsyncConfiguration.Builder}. This removes the need to call {@link ClientAsyncConfiguration#builder()} + * and {@link ClientAsyncConfiguration.Builder#build()}. + * + * @param configuration the async configuration + * @return this builder for method chaining. + * @see #asyncConfiguration(ClientAsyncConfiguration) + */ + default Builder asyncConfiguration(Consumer configuration) { + return asyncConfiguration(ClientAsyncConfiguration.builder().applyMutation(configuration).build()); + } + } + + private static final class DefaultBuilder implements Builder { + private AwsCredentialsProvider credentialsProvider; + private Region region; + private Long minimumPartSizeInBytes; + private Double targetThroughputInGbps; + private Integer maxConcurrency; + private ClientAsyncConfiguration asyncConfiguration; + + private DefaultBuilder() { + } + + private DefaultBuilder(S3ClientConfiguration configuration) { + this.credentialsProvider = configuration.credentialsProvider; + this.region = configuration.region; + this.minimumPartSizeInBytes = configuration.minimumPartSizeInBytes; + this.targetThroughputInGbps = configuration.targetThroughputInGbps; + this.maxConcurrency = configuration.maxConcurrency; + this.asyncConfiguration = configuration.asyncConfiguration; + } + + @Override + public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + @Override + public Builder region(Region region) { + this.region = region; + return this; + } + + @Override + public Builder minimumPartSizeInBytes(Long partSizeBytes) { + this.minimumPartSizeInBytes = partSizeBytes; + return this; + } + + @Override + public Builder targetThroughputInGbps(Double targetThroughputInGbps) { + this.targetThroughputInGbps = targetThroughputInGbps; + return this; + } + + @Override + public Builder maxConcurrency(Integer maxConcurrency) { + this.maxConcurrency = maxConcurrency; + return this; + } + + @Override + public Builder asyncConfiguration(ClientAsyncConfiguration asyncConfiguration) { + this.asyncConfiguration = asyncConfiguration; + return this; + } + + @Override + public S3ClientConfiguration build() { + return new S3ClientConfiguration(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java new file mode 100644 index 000000000000..6ad075aace4a --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java @@ -0,0 +1,218 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.transfer.s3.internal.DefaultS3TransferManager; +import software.amazon.awssdk.utils.SdkAutoCloseable; + +/** + * The S3 Transfer Manager is a library that allows users to easily and + * optimally upload and downloads to and from S3. + * + * Usage Example: + * + * 

+ * {@code
+ * // Create using all default configuration values
+ * S3TransferManager transferManager = S3TransferManager.create();
+ *
+ * // If you wish to configure settings, we recommend using the builder instead:
+ * S3TransferManager transferManager =
+ *                  S3TransferManager.builder()
+ *                                   .s3ClientConfiguration(b -> b.credentialsProvider(credentialProvider)
+ *                                   .region(Region.US_WEST_2)
+ *                                   .targetThroughputInGbps(20.0)
+ *                                   .minimumPartSizeInBytes(10 * MB))
+ *                                   .build();
+ *
+ * // Download an S3 object to a file
+ * Download download =
+ *     transferManager.download(b -> b.destination(Paths.get("myFile.txt"))
+ *                                    .getObjectRequest(r -> r.bucket("bucket")
+ *                                                            .key("key")));
+ * download.completionFuture().join();
+ *
+ * // Upload a file to S3
+ * Upload upload = transferManager.upload(b -> b.source(Paths.get("myFile.txt"))
+ *                                              .putObjectRequest(r -> r.bucket("bucket")
+ *                                                                      .key("key")));
+ *
+ * upload.completionFuture().join();
+ * }
+ *
+ */ +@SdkPublicApi +@SdkPreviewApi +public interface S3TransferManager extends SdkAutoCloseable { + /** + * Download an object identified by the bucket and key from S3 to the given + * file. + *

+ * Usage Example: + * 

+     * {@code
+     * // Initiate the transfer
+     * Download download =
+     *     transferManager.download(DownloadRequest.builder()
+     *                                             .destination(Paths.get("myFile.txt"))
+     *                                             .getObjectRequest(GetObjectRequest.builder()
+     *                                                                               .bucket("bucket")
+     *                                                                               .key("key")
+     *                                                                               .build())
+     *                                             .build());
+     * // Wait for the transfer to complete
+     * download.completionFuture().join();
+     * }
+     *
+ * @see #download(Consumer) + */ + default Download download(DownloadRequest downloadRequest) { + throw new UnsupportedOperationException(); + } + + /** + * Download an object identified by the bucket and key from S3 to the given + * file. + * + *

+ * This is a convenience method that creates an instance of the {@link DownloadRequest} builder avoiding the + * need to create one manually via {@link DownloadRequest#builder()}. + * + *

+ * Usage Example: + * 

+     * {@code
+     * // Initiate the transfer
+     * Download download =
+     *     transferManager.download(b -> b.destination(Paths.get("myFile.txt"))
+     *                                    .getObjectRequest(r -> r.bucket("bucket")
+     *                                                            .key("key")));
+     * // Wait for the transfer to complete
+     * download.completionFuture().join();
+     * }
+     *
+ * @see #download(DownloadRequest) + */ + default Download download(Consumer request) { + return download(DownloadRequest.builder().applyMutation(request).build()); + } + + /** + * Upload a file to S3. + *

+ * Usage Example: + * 

+     * {@code
+     * Upload upload =
+     *     transferManager.upload(UploadRequest.builder()
+     *                                        .source(Paths.get("myFile.txt"))
+     *                                        .putObjectRequest(PutObjectRequest.builder()
+     *                                                                          .bucket("bucket")
+     *                                                                          .key("key")
+     *                                                                          .build())
+     *                                        .build());
+     * // Wait for the transfer to complete
+     * upload.completionFuture().join();
+     * }
+     *
+ */ + default Upload upload(UploadRequest uploadRequest) { + throw new UnsupportedOperationException(); + } + + /** + * Upload a file to S3. + * + *

+ * This is a convenience method that creates an instance of the {@link UploadRequest} builder avoiding the + * need to create one manually via {@link UploadRequest#builder()}. + * + *

+ * Usage Example: + * 

+     * {@code
+     * Upload upload =
+     *       transferManager.upload(b -> b.putObjectRequest(req -> req.bucket("bucket")
+     *                                                                .key("key"))
+     *                                    .source(Paths.get("myFile.txt")));
+     * // Wait for the transfer to complete
+     * upload.completionFuture().join();
+     * }
+     *
+ */ + default Upload upload(Consumer request) { + return upload(UploadRequest.builder().applyMutation(request).build()); + } + + /** + * Create an {@code S3TransferManager} using the default values. + */ + static S3TransferManager create() { + return builder().build(); + } + + /** + * Creates a default builder for {@link S3TransferManager}. + */ + static S3TransferManager.Builder builder() { + return DefaultS3TransferManager.builder(); + } + + /** + * The builder definition for a {@link S3TransferManager}. + */ + interface Builder { + + /** + * Configuration values for the low level S3 client. The {@link S3TransferManager} already provides sensible + * defaults. All values are optional. + * + * @param configuration the configuration to use + * @return Returns a reference to this object so that method calls can be chained together. + * @see #s3ClientConfiguration(Consumer) + */ + Builder s3ClientConfiguration(S3ClientConfiguration configuration); + + /** + * Configuration values for the low level S3 client. The {@link S3TransferManager} already provides sensible + * defaults. All values are optional. + * + *

+ * This is a convenience method that creates an instance of the {@link S3ClientConfiguration} builder avoiding the + * need to create one manually via {@link S3ClientConfiguration#builder()}. + * + * @param configuration the configuration to use + * @return Returns a reference to this object so that method calls can be chained together. + * @see #s3ClientConfiguration(S3ClientConfiguration) + */ + default Builder s3ClientConfiguration(Consumer configuration) { + S3ClientConfiguration.Builder builder = S3ClientConfiguration.builder(); + configuration.accept(builder); + s3ClientConfiguration(builder.build()); + return this; + } + + /** + * Build an instance of {@link S3TransferManager} based on the settings supplied to this builder + * + * @return an instance of {@link S3TransferManager} + */ + S3TransferManager build(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/SizeConstant.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/SizeConstant.java new file mode 100644 index 000000000000..ad5a17abbd8f --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/SizeConstant.java @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Helpful constants for common size units. + */ +@SdkPublicApi +@SdkPreviewApi +public final class SizeConstant { + + /** + * 1 Kibibyte + * */ + public static final long KB = 1024; + + /** + * 1 Mebibyte. + */ + public static final long MB = 1024 * KB; + + /** + * 1 Gibibyte. + */ + public static final long GB = 1024 * MB; + + private SizeConstant() { + + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Transfer.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Transfer.java new file mode 100644 index 000000000000..eb91799f363a --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Transfer.java @@ -0,0 +1,32 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Represents the upload or download of one or more objects to or from S3. + */ +@SdkPublicApi +@SdkPreviewApi +public interface Transfer { + /** + * @return The future that will be completed when this transfer is complete. + */ + CompletableFuture completionFuture(); +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequest.java new file mode 100644 index 000000000000..21317003e91b --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/TransferRequest.java @@ -0,0 +1,32 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * Interface for all transfer requests. + */ +@SdkPublicApi +@SdkPreviewApi +public interface TransferRequest { + + interface Builder { + + TypeToBuildT build(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Upload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Upload.java new file mode 100644 index 000000000000..ba862ea3ba36 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/Upload.java @@ -0,0 +1,31 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; + +/** + * An upload transfer of a single object to S3. + */ +@SdkPublicApi +@SdkPreviewApi +public interface Upload extends Transfer { + @Override + CompletableFuture completionFuture(); + +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java new file mode 100644 index 000000000000..a038e172d1ba --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/UploadRequest.java @@ -0,0 +1,169 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static software.amazon.awssdk.utils.Validate.paramNotNull; + +import java.nio.file.Path; +import java.util.Objects; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.NotThreadSafe; +import software.amazon.awssdk.annotations.SdkPreviewApi; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Upload an object to S3 using {@link S3TransferManager}. + */ +@SdkPublicApi +@SdkPreviewApi +public final class UploadRequest implements TransferRequest, ToCopyableBuilder { + private final PutObjectRequest putObjectRequest; + private final Path source; + + private UploadRequest(BuilderImpl builder) { + this.putObjectRequest = paramNotNull(builder.putObjectRequest, "putObjectRequest"); + this.source = paramNotNull(builder.source, "source"); + } + + /** + * @return The {@link PutObjectRequest} request that should be used for the upload + */ + public PutObjectRequest putObjectRequest() { + return putObjectRequest; + } + + /** + * The {@link Path} to file containing data to send to the service. + * + * @return the source path + */ + public Path source() { + return source; + } + + /** + * Create a builder that can be used to create a {@link UploadRequest}. + * + * @see S3TransferManager#upload(UploadRequest) + */ + public static Builder builder() { + return new BuilderImpl(); + } + + @Override + public Builder toBuilder() { + return new BuilderImpl(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + UploadRequest that = (UploadRequest) o; + + if (!Objects.equals(putObjectRequest, that.putObjectRequest)) { + return false; + } + return Objects.equals(source, that.source); + } + + @Override + public int hashCode() { + int result = putObjectRequest != null ? putObjectRequest.hashCode() : 0; + result = 31 * result + (source != null ? source.hashCode() : 0); + return result; + } + + /** + * A builder for a {@link UploadRequest}, created with {@link #builder()} + */ + @SdkPublicApi + @NotThreadSafe + public interface Builder extends TransferRequest.Builder, CopyableBuilder { + + /** + * The {@link Path} to file containing data to send to the service. File will be read entirely and may be read + * multiple times in the event of a retry. If the file does not exist or the current user does not have + * access to read it then an exception will be thrown. + * + * @param source the source path + * @return Returns a reference to this object so that method calls can be chained together. + */ + Builder source(Path source); + + /** + * Configure the {@link PutObjectRequest} that should be used for the upload + * + * @param putObjectRequest the putObjectRequest + * @return Returns a reference to this object so that method calls can be chained together. + * @see #putObjectRequest(Consumer) + */ + Builder putObjectRequest(PutObjectRequest putObjectRequest); + + /** + * Configure the {@link PutObjectRequest} that should be used for the upload + * + *

+ * This is a convenience method that creates an instance of the {@link PutObjectRequest} builder avoiding the + * need to create one manually via {@link PutObjectRequest#builder()}. + * + * @param putObjectRequestBuilder the putObjectRequest consumer builder + * @return Returns a reference to this object so that method calls can be chained together. + * @see #putObjectRequest(PutObjectRequest) + */ + default Builder putObjectRequest(Consumer putObjectRequestBuilder) { + return putObjectRequest(PutObjectRequest.builder() + .applyMutation(putObjectRequestBuilder) + .build()); + } + + /** + * @return The built request. + */ + @Override + UploadRequest build(); + } + + private static class BuilderImpl implements Builder { + private PutObjectRequest putObjectRequest; + private Path source; + + @Override + public Builder source(Path source) { + this.source = source; + return this; + } + + @Override + public Builder putObjectRequest(PutObjectRequest putObjectRequest) { + this.putObjectRequest = putObjectRequest; + return this; + } + + @Override + public UploadRequest build() { + return new UploadRequest(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialsProviderAdapter.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialsProviderAdapter.java new file mode 100644 index 000000000000..60b1ddd7bd16 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialsProviderAdapter.java @@ -0,0 +1,68 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + + +import java.nio.charset.StandardCharsets; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.crt.auth.credentials.Credentials; +import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider; +import software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider; +import software.amazon.awssdk.utils.SdkAutoCloseable; + +/** + * Adapts an SDK {@link AwsCredentialsProvider} to CRT {@link CredentialsProvider} + */ +@SdkInternalApi +public final class CrtCredentialsProviderAdapter implements SdkAutoCloseable { + private final AwsCredentialsProvider credentialsProvider; + private final CredentialsProvider crtCredentials; + + public CrtCredentialsProviderAdapter(AwsCredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + this.crtCredentials = new DelegateCredentialsProvider.DelegateCredentialsProviderBuilder() + .withHandler(() -> { + AwsCredentials sdkCredentials = credentialsProvider.resolveCredentials(); + byte[] accessKey = sdkCredentials.accessKeyId().getBytes(StandardCharsets.UTF_8); + byte[] secreteKey = sdkCredentials.secretAccessKey().getBytes(StandardCharsets.UTF_8); + + byte[] sessionTokens = null; + if (sdkCredentials instanceof AwsSessionCredentials) { + sessionTokens = + ((AwsSessionCredentials) sdkCredentials).sessionToken().getBytes(StandardCharsets.UTF_8); + } + + return new Credentials(accessKey, + secreteKey, + sessionTokens); + }).build(); + } + + public CredentialsProvider crtCredentials() { + return crtCredentials; + } + + @Override + public void close() { + if (credentialsProvider instanceof SdkAutoCloseable) { + ((SdkAutoCloseable) credentialsProvider).close(); + } + crtCredentials.close(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandler.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandler.java new file mode 100644 index 000000000000..8c233ebbcebc --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandler.java @@ -0,0 +1,117 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.crt.s3.CrtS3RuntimeException; +import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException; +import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.NoSuchUploadException; +import software.amazon.awssdk.services.s3.model.ObjectAlreadyInActiveTierErrorException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.utils.StringUtils; + +@SdkInternalApi +public class CrtErrorHandler { + + private final Map s3ExceptionBuilderMap; + + public CrtErrorHandler() { + s3ExceptionBuilderMap = getS3ExceptionBuilderMap(); + } + + /** + * This class transform a crtTRunTimeException to the S3 service Exceptions. + * CrtS3RuntimeException are the exceptions generated due to failures in CRTClient due to S3 Service errors. + * + * @param crtRuntimeException Exception that is thrown by CrtClient. + * @return + */ + public Exception transformException(Exception crtRuntimeException) { + Optional crtS3RuntimeExceptionOptional = getCrtS3RuntimeException(crtRuntimeException); + Exception exception = crtS3RuntimeExceptionOptional + .filter(CrtErrorHandler::isErrorDetailsAvailable) + .map(e -> getServiceSideException(e)) + .orElse(SdkClientException.create(crtRuntimeException.getMessage(), crtRuntimeException)); + return exception; + } + + private Exception getServiceSideException(CrtS3RuntimeException e) { + if (s3ExceptionBuilderMap.get(e.getAwsErrorCode()) != null) { + return s3ExceptionBuilderMap.get(e.getAwsErrorCode()) + .awsErrorDetails( + AwsErrorDetails.builder().errorCode(e.getAwsErrorCode()) + .errorMessage(e.getAwsErrorMessage()).build()) + .cause(e) + .message(e.getMessage()) + .statusCode(e.getStatusCode()) + .build(); + } + return S3Exception.builder().statusCode(e.getStatusCode()).message(e.getMessage()).cause(e).build(); + } + + /** + * This method checks if the exception has the required details to transform to S3 Exception. + * @param crtS3RuntimeException the exception that needs to be checked + * @return true if exception has the required details. + */ + private static boolean isErrorDetailsAvailable(CrtS3RuntimeException crtS3RuntimeException) { + return StringUtils.isNotBlank(crtS3RuntimeException.getAwsErrorCode()); + } + + /** + * Checks if the Exception or its cause is of CrtS3RuntimeException. + * The S3 Service related exception are in the form of CrtS3RuntimeException. + * @param crtRuntimeException + * @return CrtS3RuntimeException else return empty, + */ + private Optional getCrtS3RuntimeException(Exception crtRuntimeException) { + if (crtRuntimeException instanceof CrtS3RuntimeException) { + return Optional.of((CrtS3RuntimeException) crtRuntimeException); + } + Throwable cause = crtRuntimeException.getCause(); + if (cause instanceof CrtS3RuntimeException) { + return Optional.of((CrtS3RuntimeException) cause); + } + return Optional.empty(); + } + + + /** + * Gets a Mapping of AWSErrorCode to its corresponding S3 Exception Builders. + * + * @return + */ + private Map getS3ExceptionBuilderMap() { + Map s3ExceptionBuilderMap = new HashMap<>(); + s3ExceptionBuilderMap.put("ObjectAlreadyInActiveTierError", ObjectAlreadyInActiveTierErrorException.builder()); + s3ExceptionBuilderMap.put("NoSuchUpload", NoSuchUploadException.builder()); + s3ExceptionBuilderMap.put("BucketAlreadyExists", BucketAlreadyExistsException.builder()); + s3ExceptionBuilderMap.put("BucketAlreadyOwnedByYou", BucketAlreadyOwnedByYouException.builder()); + s3ExceptionBuilderMap.put("InvalidObjectState", InvalidObjectStateException.builder()); + s3ExceptionBuilderMap.put("NoSuchBucket", NoSuchBucketException.builder()); + s3ExceptionBuilderMap.put("NoSuchKey", NoSuchKeyException.builder()); + return s3ExceptionBuilderMap; + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapter.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapter.java new file mode 100644 index 000000000000..2c6632d7d0c0 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapter.java @@ -0,0 +1,99 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import com.amazonaws.s3.ResponseDataConsumer; +import com.amazonaws.s3.model.GetObjectOutput; +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.crt.CrtRuntimeException; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.utils.Logger; + +/** + * Adapt the SDK API {@link AsyncResponseTransformer} to the CRT API {@link ResponseDataConsumer}. + */ +@SdkInternalApi +public class CrtResponseDataConsumerAdapter implements ResponseDataConsumer { + + private static final Logger log = Logger.loggerFor(CrtResponseDataConsumerAdapter.class); + private final AsyncResponseTransformer transformer; + private final CompletableFuture future; + private final S3CrtDataPublisher publisher; + private final ResponseHeadersHandler headerHandler; + private final CrtErrorHandler errorHandler; + + + public CrtResponseDataConsumerAdapter(AsyncResponseTransformer transformer) { + this(transformer, new S3CrtDataPublisher(), new ResponseHeadersHandler()); + } + + @SdkInternalApi + CrtResponseDataConsumerAdapter(AsyncResponseTransformer transformer, + S3CrtDataPublisher s3CrtDataPublisher, + ResponseHeadersHandler headersHandler) { + this.transformer = transformer; + this.future = transformer.prepare(); + this.publisher = s3CrtDataPublisher; + this.headerHandler = headersHandler; + this.errorHandler = new CrtErrorHandler(); + } + + public CompletableFuture transformerFuture() { + return future; + } + + @Override + public void onResponseHeaders(int statusCode, HttpHeader[] headers) { + headerHandler.onResponseHeaders(statusCode, headers); + } + + @Override + public void onResponse(GetObjectOutput output) { + // Passing empty SdkHttpResponse if it's not available + SdkHttpResponse sdkHttpResponse = headerHandler.sdkHttpResponseFuture() + .getNow(SdkHttpResponse.builder().build()); + + GetObjectResponse response = S3CrtPojoConversion.fromCrtGetObjectOutput(output, + sdkHttpResponse); + transformer.onResponse(response); + transformer.onStream(publisher); + } + + @Override + public void onResponseData(ByteBuffer byteBuffer) { + log.trace(() -> "Received data of size " + byteBuffer.remaining()); + publisher.deliverData(byteBuffer); + } + + @Override + public void onException(CrtRuntimeException e) { + log.debug(() -> "An error occurred ", e); + Exception transformException = errorHandler.transformException(e); + transformer.exceptionOccurred(transformException); + publisher.notifyError(transformException); + } + + @Override + public void onFinished() { + log.debug(() -> "Finished streaming "); + publisher.notifyStreamingFinished(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedDownload.java new file mode 100644 index 000000000000..46a9716f61bd --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedDownload.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.transfer.s3.CompletedDownload; + +@SdkInternalApi +public final class DefaultCompletedDownload implements CompletedDownload { + private final GetObjectResponse response; + + private DefaultCompletedDownload(Builder builder) { + this.response = builder.response; + } + + @Override + public GetObjectResponse response() { + return response; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private GetObjectResponse response; + + public Builder response(GetObjectResponse response) { + this.response = response; + return this; + } + + public CompletedDownload build() { + return new DefaultCompletedDownload(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedUpload.java new file mode 100644 index 000000000000..3d13b87c56bf --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultCompletedUpload.java @@ -0,0 +1,62 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.transfer.s3.CompletedUpload; + +@SdkInternalApi +public final class DefaultCompletedUpload implements CompletedUpload { + private final PutObjectResponse response; + + private DefaultCompletedUpload(BuilderImpl builder) { + this.response = builder.response; + } + + @Override + public PutObjectResponse response() { + return response; + } + + /** + * Creates a default builder for {@link CompletedUpload}. + */ + public static Builder builder() { + return new DefaultCompletedUpload.BuilderImpl(); + } + + interface Builder { + Builder response(PutObjectResponse response); + + CompletedUpload build(); + } + + private static class BuilderImpl implements Builder { + private PutObjectResponse response; + + @Override + public Builder response(PutObjectResponse response) { + this.response = response; + return this; + } + + @Override + public CompletedUpload build() { + return new DefaultCompletedUpload(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownload.java new file mode 100644 index 000000000000..8d112c4a3190 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultDownload.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.transfer.s3.CompletedDownload; +import software.amazon.awssdk.transfer.s3.Download; + +@SdkInternalApi +public final class DefaultDownload implements Download { + private final CompletableFuture completionFuture; + + public DefaultDownload(CompletableFuture completionFuture) { + this.completionFuture = completionFuture; + } + + @Override + public CompletableFuture completionFuture() { + return completionFuture; + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java new file mode 100644 index 000000000000..5bc122bcb5d0 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java @@ -0,0 +1,227 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import com.amazonaws.s3.S3NativeClient; +import com.amazonaws.s3.model.GetObjectOutput; +import com.amazonaws.s3.model.PutObjectOutput; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.utils.CompletableFutureUtils; + +@SdkInternalApi +public final class DefaultS3CrtAsyncClient implements S3CrtAsyncClient { + private final S3NativeClient s3NativeClient; + private final S3NativeClientConfiguration configuration; + private final CrtErrorHandler crtErrorHandler; + + public DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) { + S3NativeClientConfiguration.Builder configBuilder = + S3NativeClientConfiguration.builder() + .targetThroughputInGbps(builder.targetThroughputInGbps()) + .partSizeInBytes(builder.minimumPartSizeInBytes()) + .maxConcurrency(builder.maxConcurrency) + .credentialsProvider(builder.credentialsProvider) + .asyncConfiguration(builder.asyncConfiguration); + if (builder.region() != null) { + configBuilder.signingRegion(builder.region().id()); + } + + configuration = configBuilder.build(); + + this.s3NativeClient = new S3NativeClient(configuration.signingRegion(), + configuration.clientBootstrap(), + configuration.credentialsProvider(), + configuration.partSizeBytes(), + configuration.targetThroughputInGbps(), + configuration.maxConcurrency()); + this.crtErrorHandler = new CrtErrorHandler(); + } + + @SdkTestInternalApi + DefaultS3CrtAsyncClient(S3NativeClientConfiguration configuration, + S3NativeClient nativeClient) { + this.configuration = configuration; + this.s3NativeClient = nativeClient; + this.crtErrorHandler = new CrtErrorHandler(); + + } + + @Override + public CompletableFuture getObject( + GetObjectRequest getObjectRequest, AsyncResponseTransformer asyncResponseTransformer) { + + CompletableFuture returnFuture = new CompletableFuture<>(); + com.amazonaws.s3.model.GetObjectRequest crtGetObjectRequest = S3CrtPojoConversion.toCrtGetObjectRequest(getObjectRequest); + CrtResponseDataConsumerAdapter adapter = new CrtResponseDataConsumerAdapter<>(asyncResponseTransformer); + + CompletableFuture adapterFuture = adapter.transformerFuture(); + + CompletableFuture crtFuture = s3NativeClient.getObject(crtGetObjectRequest, adapter); + + // Forward the cancellation to crtFuture to cancel the request + CompletableFutureUtils.forwardExceptionTo(returnFuture, crtFuture); + + + // Forward the exception from the CRT future to the return future in case + // the adapter callback didn't get it + CompletableFutureUtils.forwardTransformedExceptionTo(crtFuture, returnFuture, + t -> t instanceof Exception ? crtErrorHandler.transformException((Exception) t) : t); + + returnFuture.whenComplete((r, t) -> { + if (t == null) { + returnFuture.complete(r); + } else { + returnFuture.completeExceptionally(t instanceof Exception + ? crtErrorHandler.transformException((Exception) t) : t); + } + }); + + CompletableFutureUtils.forwardResultTo(adapterFuture, returnFuture, configuration.futureCompletionExecutor()); + + return CompletableFutureUtils.forwardExceptionTo(returnFuture, adapterFuture); + } + + @Override + public CompletableFuture putObject(PutObjectRequest putObjectRequest, AsyncRequestBody requestBody) { + CompletableFuture returnFuture = new CompletableFuture<>(); + + com.amazonaws.s3.model.PutObjectRequest adaptedRequest = S3CrtPojoConversion.toCrtPutObjectRequest(putObjectRequest); + + if (adaptedRequest.contentLength() == null && requestBody.contentLength().isPresent()) { + adaptedRequest = adaptedRequest.toBuilder() + .contentLength(requestBody.contentLength().get()) + .build(); + } + + RequestDataSupplierAdapter requestDataSupplier = new RequestDataSupplierAdapter(requestBody); + CompletableFuture crtFuture = s3NativeClient.putObject(adaptedRequest, + requestDataSupplier); + // Forward the cancellation to crtFuture to cancel the request + CompletableFutureUtils.forwardExceptionTo(returnFuture, crtFuture); + + CompletableFuture httpResponseFuture = requestDataSupplier.sdkHttpResponseFuture(); + CompletableFuture executeFuture = + // If the header is not available, passing empty SDK HTTP response + crtFuture.thenApply(putObjectOutput -> S3CrtPojoConversion.fromCrtPutObjectOutput( + putObjectOutput, httpResponseFuture.getNow(SdkHttpResponse.builder().build()))); + + executeFuture.whenComplete((r, t) -> { + if (t == null) { + returnFuture.complete(r); + } else { + returnFuture.completeExceptionally(t instanceof Exception + ? crtErrorHandler.transformException((Exception) t) : t); + } + }); + + CompletableFutureUtils.forwardResultTo(executeFuture, returnFuture, configuration.futureCompletionExecutor()); + + return CompletableFutureUtils.forwardExceptionTo(returnFuture, executeFuture); + } + + @Override + public String serviceName() { + return "s3"; + } + + @Override + public void close() { + s3NativeClient.close(); + configuration.close(); + } + + public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientBuilder { + private AwsCredentialsProvider credentialsProvider; + private Region region; + private Long minimalPartSizeInBytes; + private Double targetThroughputInGbps; + private Integer maxConcurrency; + private ClientAsyncConfiguration asyncConfiguration; + + public AwsCredentialsProvider credentialsProvider() { + return credentialsProvider; + } + + public Region region() { + return region; + } + + public Long minimumPartSizeInBytes() { + return minimalPartSizeInBytes; + } + + public Double targetThroughputInGbps() { + return targetThroughputInGbps; + } + + public Integer maxConcurrency() { + return maxConcurrency; + } + + @Override + public S3CrtAsyncClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + @Override + public S3CrtAsyncClientBuilder region(Region region) { + this.region = region; + return this; + } + + @Override + public S3CrtAsyncClientBuilder minimumPartSizeInBytes(Long partSizeBytes) { + this.minimalPartSizeInBytes = partSizeBytes; + return this; + } + + @Override + public S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInGbps) { + this.targetThroughputInGbps = targetThroughputInGbps; + return this; + } + + @Override + public S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency) { + this.maxConcurrency = maxConcurrency; + return this; + } + + @Override + public S3CrtAsyncClientBuilder asyncConfiguration(ClientAsyncConfiguration configuration) { + this.asyncConfiguration = configuration; + return this; + } + + @Override + public S3CrtAsyncClient build() { + return new DefaultS3CrtAsyncClient(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java new file mode 100644 index 000000000000..55fcb723685e --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java @@ -0,0 +1,108 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.transfer.s3.CompletedDownload; +import software.amazon.awssdk.transfer.s3.CompletedUpload; +import software.amazon.awssdk.transfer.s3.Download; +import software.amazon.awssdk.transfer.s3.DownloadRequest; +import software.amazon.awssdk.transfer.s3.S3ClientConfiguration; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.Upload; +import software.amazon.awssdk.transfer.s3.UploadRequest; +import software.amazon.awssdk.utils.CompletableFutureUtils; + +@SdkInternalApi +public final class DefaultS3TransferManager implements S3TransferManager { + private final S3CrtAsyncClient s3CrtAsyncClient; + + public DefaultS3TransferManager(DefaultBuilder builder) { + S3CrtAsyncClient.S3CrtAsyncClientBuilder clientBuilder = S3CrtAsyncClient.builder(); + builder.s3ClientConfiguration.credentialsProvider().ifPresent(clientBuilder::credentialsProvider); + builder.s3ClientConfiguration.maxConcurrency().ifPresent(clientBuilder::maxConcurrency); + builder.s3ClientConfiguration.minimumPartSizeInBytes().ifPresent(clientBuilder::minimumPartSizeInBytes); + builder.s3ClientConfiguration.region().ifPresent(clientBuilder::region); + builder.s3ClientConfiguration.targetThroughputInGbps().ifPresent(clientBuilder::targetThroughputInGbps); + builder.s3ClientConfiguration.asyncConfiguration().ifPresent(clientBuilder::asyncConfiguration); + + s3CrtAsyncClient = clientBuilder.build(); + } + + @SdkTestInternalApi + DefaultS3TransferManager(S3CrtAsyncClient s3CrtAsyncClient) { + this.s3CrtAsyncClient = s3CrtAsyncClient; + } + + @Override + public Upload upload(UploadRequest uploadRequest) { + PutObjectRequest putObjectRequest = uploadRequest.putObjectRequest(); + AsyncRequestBody requestBody = requestBodyFor(uploadRequest); + + CompletableFuture putObjFuture = s3CrtAsyncClient.putObject(putObjectRequest, requestBody); + + CompletableFuture future = putObjFuture.thenApply(r -> DefaultCompletedUpload.builder() + .response(r) + .build()); + return new DefaultUpload(CompletableFutureUtils.forwardExceptionTo(future, putObjFuture)); + } + + @Override + public Download download(DownloadRequest downloadRequest) { + CompletableFuture getObjectFuture = + s3CrtAsyncClient.getObject(downloadRequest.getObjectRequest(), + AsyncResponseTransformer.toFile(downloadRequest.destination())); + CompletableFuture future = + getObjectFuture.thenApply(r -> DefaultCompletedDownload.builder().response(r).build()); + + return new DefaultDownload(CompletableFutureUtils.forwardExceptionTo(future, getObjectFuture)); + } + + @Override + public void close() { + s3CrtAsyncClient.close(); + } + + public static Builder builder() { + return new DefaultBuilder(); + } + + private AsyncRequestBody requestBodyFor(UploadRequest uploadRequest) { + return AsyncRequestBody.fromFile(uploadRequest.source()); + } + + private static class DefaultBuilder implements S3TransferManager.Builder { + private S3ClientConfiguration s3ClientConfiguration; + + @Override + public Builder s3ClientConfiguration(S3ClientConfiguration configuration) { + this.s3ClientConfiguration = configuration; + return this; + } + + @Override + public S3TransferManager build() { + return new DefaultS3TransferManager(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultUpload.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultUpload.java new file mode 100644 index 000000000000..4ac6dbe7e78f --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultUpload.java @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.transfer.s3.CompletedUpload; +import software.amazon.awssdk.transfer.s3.Upload; + +@SdkInternalApi +public final class DefaultUpload implements Upload { + private final CompletableFuture completionFeature; + + public DefaultUpload(CompletableFuture completionFeature) { + this.completionFeature = completionFeature; + } + + @Override + public CompletableFuture completionFuture() { + return completionFeature; + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapter.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapter.java new file mode 100644 index 000000000000..14cbecb4f29b --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapter.java @@ -0,0 +1,367 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import com.amazonaws.s3.RequestDataSupplier; +import java.nio.ByteBuffer; +import java.util.Deque; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import org.reactivestreams.Publisher; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.crt.CrtRuntimeException; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.utils.Logger; + +/** + * Adapts an SDK {@link software.amazon.awssdk.core.async.AsyncRequestBody} to CRT's {@link RequestDataSupplier}. + */ +@SdkInternalApi +public final class RequestDataSupplierAdapter implements RequestDataSupplier { + static final long DEFAULT_REQUEST_SIZE = 8; + private static final Logger LOG = Logger.loggerFor(RequestDataSupplierAdapter.class); + + private final AtomicReference subscriptionStatus = + new AtomicReference<>(SubscriptionStatus.NOT_SUBSCRIBED); + private final BlockingQueue subscriptionQueue = new LinkedBlockingQueue<>(1); + private final BlockingDeque eventBuffer = new LinkedBlockingDeque<>(); + + private final Publisher bodyPublisher; + + private volatile Subscription subscription; + + // TODO: not volatile since it's read and written only by CRT thread(s). Need to + // ensure that CRT actually ensures consistency across their threads... + private Subscriber subscriber; + private long pending = 0; + private final ResponseHeadersHandler headersHandler; + + public RequestDataSupplierAdapter(Publisher bodyPublisher) { + this.bodyPublisher = bodyPublisher; + this.subscriber = createSubscriber(); + this.headersHandler = new ResponseHeadersHandler(); + } + + public CompletableFuture sdkHttpResponseFuture() { + return headersHandler.sdkHttpResponseFuture(); + } + + @Override + public void onResponseHeaders(final int statusCode, final HttpHeader[] headers) { + headersHandler.onResponseHeaders(statusCode, headers); + } + + @Override + public boolean getRequestBytes(ByteBuffer outBuffer) { + LOG.trace(() -> "Getting data to fill buffer of size " + outBuffer.remaining()); + + // Per the spec, onSubscribe is always called before any other + // signal, so we expect a subscription to always be provided; we just + // wait for that to happen + waitForSubscription(); + + // The "event loop". Per the spec, the sequence of events is "onSubscribe onNext* (onError | onComplete)?". + // We don't handle onSubscribe as a discrete event; instead we only enter this loop once we have a + // subscription. + // + // This works by requesting and consuming DATA events until we fill the buffer. We return from the method if + // we encounter either of the terminal events, COMPLETE or ERROR. + while (true) { + // The supplier API requires that we fill the buffer entirely. + if (!outBuffer.hasRemaining()) { + break; + } + + if (eventBuffer.isEmpty() && pending == 0) { + pending = DEFAULT_REQUEST_SIZE; + subscription.request(pending); + } + + Event ev = takeFirstEvent(); + + // Discard the event if it's not for the current subscriber + if (!ev.subscriber().equals(subscriber)) { + LOG.debug(() -> "Received an event for a previous publisher. Discarding. Event was: " + ev); + continue; + } + + switch (ev.type()) { + case DATA: + ByteBuffer srcBuffer = ((DataEvent) ev).data(); + + ByteBuffer bufferToWrite = srcBuffer.duplicate(); + int nBytesToWrite = Math.min(outBuffer.remaining(), srcBuffer.remaining()); + + // src is larger, create a resized view to prevent + // buffer overflow in the subsequent put() call + if (bufferToWrite.remaining() > nBytesToWrite) { + bufferToWrite.limit(bufferToWrite.position() + nBytesToWrite); + } + + outBuffer.put(bufferToWrite); + srcBuffer.position(bufferToWrite.limit()); + + if (!srcBuffer.hasRemaining()) { + --pending; + } else { + eventBuffer.push(ev); + } + + break; + + case COMPLETE: + // Leave this event in the queue so that if getRequestData + // gets call after the stream is already done, we pop it off again. + eventBuffer.push(ev); + pending = 0; + return true; + + case ERROR: + // Leave this event in the queue so that if getRequestData + // gets call after the stream is already done, we pop it off again. + eventBuffer.push(ev); + Throwable t = ((ErrorEvent) ev).error(); + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } + throw new RuntimeException(t); + + default: + // In case new event types are introduced that this loop doesn't account for + throw new IllegalStateException("Unknown event type: " + ev.type()); + } + } + + return false; + } + + @Override + public boolean resetPosition() { + subscription.cancel(); + subscription = null; + + this.subscriber = createSubscriber(); + subscriptionStatus.set(SubscriptionStatus.NOT_SUBSCRIBED); + + // NOTE: It's possible that even after this happens, eventBuffer gets + // residual events from the canceled subscription if the publisher + // handles cancel asynchronously. That doesn't affect us too much since + // we always ensure the event is for the current subscriber. + eventBuffer.clear(); + pending = 0; + + return true; + } + + @Override + public void onException(CrtRuntimeException e) { + if (subscription != null) { + subscription.cancel(); + } + } + + @Override + public void onFinished() { + if (subscription != null) { + subscription.cancel(); + } + } + + private Event takeFirstEvent() { + try { + return eventBuffer.takeFirst(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting for next event", e); + } + } + + public SubscriberImpl createSubscriber() { + return new SubscriberImpl(this::setSubscription, eventBuffer); + } + + private void setSubscription(Subscription subscription) { + if (subscriptionStatus.compareAndSet(SubscriptionStatus.SUBSCRIBING, SubscriptionStatus.SUBSCRIBED)) { + subscriptionQueue.add(subscription); + } else { + LOG.error(() -> "The supplier stopped waiting for the subscription. This is likely because it took " + + "longer than the timeout to arrive. Cancelling the subscription"); + subscription.cancel(); + } + } + + static class SubscriberImpl implements Subscriber { + private final Consumer subscriptionSetter; + private final Deque eventBuffer; + private boolean subscribed = false; + + SubscriberImpl(Consumer subscriptionSetter, Deque eventBuffer) { + this.subscriptionSetter = subscriptionSetter; + this.eventBuffer = eventBuffer; + } + + @Override + public void onSubscribe(Subscription subscription) { + if (subscription == null) { + throw new NullPointerException("Subscription must not be null"); + } + + if (subscribed) { + subscription.cancel(); + return; + } + + subscriptionSetter.accept(subscription); + subscribed = true; + } + + @Override + public void onNext(ByteBuffer byteBuffer) { + if (byteBuffer == null) { + throw new NullPointerException("byteBuffer must not be null"); + } + LOG.trace(() -> "Received new data of size: " + byteBuffer.remaining()); + eventBuffer.add(new DataEvent(this, byteBuffer)); + } + + @Override + public void onError(Throwable throwable) { + eventBuffer.add(new ErrorEvent(this, throwable)); + } + + @Override + public void onComplete() { + eventBuffer.add(new CompleteEvent(this)); + } + } + + private void waitForSubscription() { + if (!subscriptionStatus.compareAndSet(SubscriptionStatus.NOT_SUBSCRIBED, SubscriptionStatus.SUBSCRIBING)) { + return; + } + + bodyPublisher.subscribe(this.subscriber); + + try { + this.subscription = subscriptionQueue.poll(5, TimeUnit.SECONDS); + if (subscription == null) { + if (!subscriptionStatus.compareAndSet(SubscriptionStatus.SUBSCRIBING, SubscriptionStatus.TIMED_OUT)) { + subscriptionQueue.take().cancel(); + } + + throw new RuntimeException("Publisher did not respond with a subscription within 5 seconds"); + } + } catch (InterruptedException e) { + LOG.error(() -> "Interrupted while waiting for subscription", e); + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting for subscription", e); + } + } + + private enum EventType { + DATA, + COMPLETE, + ERROR + } + + interface Event { + Subscriber subscriber(); + + EventType type(); + } + + private static final class DataEvent implements Event { + private final Subscriber subscriber; + private final ByteBuffer data; + + DataEvent(Subscriber subscriber, ByteBuffer data) { + this.subscriber = subscriber; + this.data = data; + } + + @Override + public Subscriber subscriber() { + return subscriber; + } + + @Override + public EventType type() { + return EventType.DATA; + } + + public ByteBuffer data() { + return data; + } + } + + private static final class CompleteEvent implements Event { + private final Subscriber subscriber; + + CompleteEvent(Subscriber subscriber) { + this.subscriber = subscriber; + } + + @Override + public Subscriber subscriber() { + return subscriber; + } + + @Override + public EventType type() { + return EventType.COMPLETE; + } + } + + private static final class ErrorEvent implements Event { + private final Subscriber subscriber; + private final Throwable error; + + ErrorEvent(Subscriber subscriber, Throwable error) { + this.subscriber = subscriber; + this.error = error; + } + + @Override + public Subscriber subscriber() { + return subscriber; + } + + @Override + public EventType type() { + return EventType.ERROR; + } + + public Throwable error() { + return error; + } + } + + private enum SubscriptionStatus { + NOT_SUBSCRIBED, + SUBSCRIBING, + SUBSCRIBED, + TIMED_OUT + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandler.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandler.java new file mode 100644 index 000000000000..5f5d7193bcef --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandler.java @@ -0,0 +1,63 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZ_ID_2_HEADER; + +import com.amazonaws.s3.OperationHandler; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkStandardLogger; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.http.HttpStatusFamily; +import software.amazon.awssdk.http.SdkHttpResponse; + +@SdkInternalApi +public final class ResponseHeadersHandler implements OperationHandler { + private static final String REQUEST_ID = "x-amz-request-id"; + private final SdkHttpResponse.Builder responseBuilder; + private final CompletableFuture responseFuture; + + public ResponseHeadersHandler() { + responseBuilder = SdkHttpResponse.builder(); + responseFuture = new CompletableFuture<>(); + } + + @Override + public void onResponseHeaders(int statusCode, HttpHeader[] headers) { + if (HttpStatusFamily.of(statusCode) == HttpStatusFamily.SUCCESSFUL) { + SdkStandardLogger.REQUEST_LOGGER.debug(() -> "Received successful response: " + statusCode); + } else { + SdkStandardLogger.REQUEST_LOGGER.debug(() -> "Received error response: " + statusCode); + } + + for (HttpHeader header : headers) { + responseBuilder.appendHeader(header.getName(), header.getValue()); + } + responseBuilder.statusCode(statusCode); + SdkStandardLogger.REQUEST_ID_LOGGER.debug(() -> REQUEST_ID + " : " + + responseBuilder.firstMatchingHeader(REQUEST_ID) + .orElse("not available")); + SdkStandardLogger.REQUEST_ID_LOGGER.debug(() -> X_AMZ_ID_2_HEADER + " : " + + responseBuilder.firstMatchingHeader(X_AMZ_ID_2_HEADER) + .orElse("not available")); + responseFuture.complete(responseBuilder.build()); + } + + public CompletableFuture sdkHttpResponseFuture() { + return responseFuture; + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java new file mode 100644 index 000000000000..097313ed67c1 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java @@ -0,0 +1,68 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.utils.builder.SdkBuilder; + +/** + * Service client for accessing Amazon S3 asynchronously using the AWS Common Runtime S3 client. This can be created using the + * static {@link #builder()} method. + */ +@SdkInternalApi +public interface S3CrtAsyncClient extends S3AsyncClient { + + interface S3CrtAsyncClientBuilder extends SdkBuilder { + S3CrtAsyncClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider); + + S3CrtAsyncClientBuilder region(Region region); + + S3CrtAsyncClientBuilder minimumPartSizeInBytes(Long uploadPartSize); + + S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInGbps); + + S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency); + + /** + * Specify overrides to the default SDK async configuration that should be used for clients created by this builder. + */ + S3CrtAsyncClientBuilder asyncConfiguration(ClientAsyncConfiguration configuration); + + /** + * Similar to {@link #asyncConfiguration(ClientAsyncConfiguration)}, but takes a lambda to configure a new + * {@link ClientAsyncConfiguration.Builder}. This removes the need to called {@link ClientAsyncConfiguration#builder()} + * and {@link ClientAsyncConfiguration.Builder#build()}. + */ + default S3CrtAsyncClientBuilder asyncConfiguration(Consumer clientAsyncConfiguration) { + return asyncConfiguration(ClientAsyncConfiguration.builder().applyMutation(clientAsyncConfiguration).build()); + } + + @Override + S3CrtAsyncClient build(); + } + + /** + * Create a builder that can be used to configure and create a {@link S3AsyncClient}. + */ + static S3CrtAsyncClientBuilder builder() { + return new DefaultS3CrtAsyncClient.DefaultS3CrtClientBuilder(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisher.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisher.java new file mode 100644 index 000000000000..b1f3a4f2e6ca --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisher.java @@ -0,0 +1,267 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.nio.ByteBuffer; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.utils.Logger; + +/** + * Publisher of the response data from crt. Tracks outstanding demand and delivers the data to the subscriber + */ +@SdkInternalApi +public class S3CrtDataPublisher implements SdkPublisher { + private static final Logger log = Logger.loggerFor(S3CrtDataPublisher.class); + private static final Event COMPLETE = new CompleteEvent(); + private static final Event CANCEL = new CancelEvent(); + /** + * Flag to indicate we are currently delivering events to the subscriber. + */ + private final AtomicBoolean isDelivering = new AtomicBoolean(false); + private final Queue buffer = new ConcurrentLinkedQueue<>(); + private final AtomicLong outstandingDemand = new AtomicLong(0); + private final AtomicReference> subscriberRef = new AtomicReference<>(null); + + private volatile boolean isDone; + + @Override + public void subscribe(Subscriber subscriber) { + if (subscriberRef.compareAndSet(null, subscriber)) { + subscriber.onSubscribe(new DataSubscription()); + + // Per Reactive-Streams spec 104, if a Publisher fails it MUST signal an onError. + notifyErrorIfNeeded(subscriber); + } else { + log.error(() -> "DataPublisher can only be subscribed to once."); + throw new IllegalStateException("DataPublisher may only be subscribed to once"); + } + } + + public void notifyStreamingFinished() { + // If the subscription is cancelled, no op + if (isDone) { + return; + } + + buffer.add(COMPLETE); + flushBuffer(); + } + + public void notifyError(Exception exception) { + // If the subscription is cancelled, no op + if (isDone) { + return; + } + + isDone = true; + buffer.clear(); + buffer.add(new ErrorEvent(exception)); + flushBuffer(); + } + + public void deliverData(ByteBuffer byteBuffer) { + // If the subscription is cancelled, no op + if (isDone) { + return; + } + buffer.add(new DataEvent(byteBuffer)); + flushBuffer(); + } + + private void notifyErrorIfNeeded(Subscriber subscriber) { + Event event = buffer.peek(); + if (event != null && event.type().equals(EventType.ERROR)) { + isDone = true; + subscriber.onError(((ErrorEvent) event).error()); + } + } + + private boolean isTerminalEvent(Event event) { + return event.type().equals(EventType.ERROR) || + event.type().equals(EventType.COMPLETE) || + event.type().equals(EventType.CANCEL); + } + + private void handleTerminalEvent(Event event) { + switch (event.type()) { + case COMPLETE: + isDone = true; + subscriberRef.get().onComplete(); + break; + case ERROR: + ErrorEvent errorEvent = (ErrorEvent) event; + subscriberRef.get().onError(errorEvent.error()); + break; + case CANCEL: + subscriberRef.set(null); + break; + default: + throw new IllegalStateException("Unexpected value: " + event.type()); + } + } + + private void flushBuffer() { + if (buffer.isEmpty()) { + return; + } + // if it's already draining, no op + if (subscriberRef.get() != null && isDelivering.compareAndSet(false, true)) { + + // If it's a terminal event, then we don't to check if there's an outstandingDemand + Event firstEvent = buffer.peek(); + if (firstEvent != null && isTerminalEvent(firstEvent)) { + Event terminalEvent = buffer.poll(); + handleTerminalEvent(terminalEvent); + isDelivering.set(false); + return; + } + + while (!buffer.isEmpty() && outstandingDemand.get() > 0) { + log.trace(() -> "Publishing data, buffer size: " + buffer.size() + ", demand: " + outstandingDemand.get()); + Event event = buffer.poll(); + // It's possible that the buffer gets cleared in notifyError() or cancel() and the subscriber + // gets cleared in cancel() + if (event == null || subscriberRef.get() == null) { + break; + } + + if (isTerminalEvent(event)) { + handleTerminalEvent(event); + isDelivering.set(false); + return; + } + + DataEvent dataEvent = (DataEvent) event; + outstandingDemand.decrementAndGet(); + subscriberRef.get().onNext(dataEvent.data()); + } + isDelivering.set(false); + } + } + + private final class DataSubscription implements Subscription { + + @Override + public void request(long n) { + if (isDone) { + return; + } + + if (n <= 0) { + subscriberRef.get().onError(new IllegalArgumentException("Request is for <= 0 elements: " + n)); + return; + } + + addDemand(n); + log.trace(() -> "Received demand: " + n + ". Total demands: " + outstandingDemand.get()); + flushBuffer(); + } + + @Override + public void cancel() { + if (isDone) { + return; + } + + log.debug(() -> "The subscription is cancelled"); + isDone = true; + buffer.clear(); + buffer.add(CANCEL); + flushBuffer(); + } + + private void addDemand(long n) { + + outstandingDemand.getAndUpdate(initialDemand -> { + if (Long.MAX_VALUE - initialDemand < n) { + return Long.MAX_VALUE; + } else { + return initialDemand + n; + } + }); + } + } + + private enum EventType { + DATA, + COMPLETE, + ERROR, + CANCEL + } + + private interface Event { + EventType type(); + } + + private static final class DataEvent implements Event { + private final ByteBuffer data; + + DataEvent(ByteBuffer data) { + this.data = data; + } + + @Override + public EventType type() { + return EventType.DATA; + } + + public ByteBuffer data() { + return data; + } + } + + private static final class CompleteEvent implements Event { + + @Override + public EventType type() { + return EventType.COMPLETE; + } + } + + private static final class CancelEvent implements Event { + + @Override + public EventType type() { + return EventType.CANCEL; + } + } + + private static class ErrorEvent implements Event { + private final Throwable error; + + ErrorEvent(Throwable error) { + this.error = error; + } + + @Override + public EventType type() { + return EventType.ERROR; + } + + public final Throwable error() { + return error; + } + } +} + diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversion.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversion.java new file mode 100644 index 000000000000..8e8c7d6cb812 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversion.java @@ -0,0 +1,309 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import com.amazonaws.s3.model.GetObjectOutput; +import com.amazonaws.s3.model.ObjectCannedACL; +import com.amazonaws.s3.model.ObjectLockLegalHoldStatus; +import com.amazonaws.s3.model.ObjectLockMode; +import com.amazonaws.s3.model.PutObjectOutput; +import com.amazonaws.s3.model.RequestPayer; +import com.amazonaws.s3.model.ServerSideEncryption; +import com.amazonaws.s3.model.StorageClass; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.awscore.DefaultAwsResponseMetadata; +import software.amazon.awssdk.core.util.SdkUserAgent; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3ResponseMetadata; +import software.amazon.awssdk.utils.http.SdkHttpUtils; + +/** + * Helper class to convert CRT POJOs to SDK POJOs and vice versa + */ +//TODO: codegen this class in the future +@SdkInternalApi +public final class S3CrtPojoConversion { + private static final String HEADER_USER_AGENT = "User-Agent"; + private static final String USER_AGENT_STRING = SdkUserAgent.create().userAgent() + " ft/s3-transfer"; + + private S3CrtPojoConversion() { + } + + public static com.amazonaws.s3.model.GetObjectRequest toCrtGetObjectRequest(GetObjectRequest request) { + com.amazonaws.s3.model.GetObjectRequest.Builder getObjectBuilder = + com.amazonaws.s3.model.GetObjectRequest.builder() + .bucket(request.bucket()) + .ifMatch(request.ifMatch()) + .ifModifiedSince(request.ifModifiedSince()) + .ifNoneMatch(request.ifNoneMatch()) + .ifUnmodifiedSince(request.ifUnmodifiedSince()) + .key(request.key()) + .range(request.range()) + .responseCacheControl(request.responseCacheControl()) + .responseContentDisposition(request.responseContentDisposition()) + .responseContentEncoding(request.responseContentEncoding()) + .responseContentLanguage(request.responseContentLanguage()) + .responseContentType(request.responseContentType()) + .responseExpires(request.responseExpires()) + .versionId(request.versionId()) + .sSECustomerAlgorithm(request.sseCustomerAlgorithm()) + .sSECustomerKey(request.sseCustomerKey()) + .sSECustomerKeyMD5(request.sseCustomerKeyMD5()) + .requestPayer(RequestPayer.fromValue(request.requestPayerAsString())) + .partNumber(request.partNumber()) + .expectedBucketOwner(request.expectedBucketOwner()); + + processRequestOverrideConfiguration(request.overrideConfiguration().orElse(null), + getObjectBuilder::customQueryParameters); + + addCustomHeaders(request.overrideConfiguration().orElse(null), getObjectBuilder::customHeaders); + + return getObjectBuilder.build(); + + } + + public static GetObjectResponse fromCrtGetObjectOutput(GetObjectOutput response, SdkHttpResponse sdkHttpResponse) { + S3ResponseMetadata s3ResponseMetadata = createS3ResponseMetadata(sdkHttpResponse); + + GetObjectResponse.Builder builder = GetObjectResponse.builder() + .deleteMarker(response.deleteMarker()) + .acceptRanges(response.acceptRanges()) + .expiration(response.expiration()) + .restore(response.restore()) + .lastModified(response.lastModified()) + .contentLength(response.contentLength()) + .eTag(response.eTag()) + .missingMeta(response.missingMeta()) + .versionId(response.versionId()) + .cacheControl(response.cacheControl()) + .contentDisposition(response.contentDisposition()) + .contentEncoding(response.contentEncoding()) + .contentLanguage(response.contentLanguage()) + .contentRange(response.contentRange()) + .contentType(response.contentType()) + .expires(response.expires()) + .websiteRedirectLocation(response.websiteRedirectLocation()) + .metadata(response.metadata()) + .sseCustomerAlgorithm(response.sSECustomerAlgorithm()) + .sseCustomerKeyMD5(response.sSECustomerKeyMD5()) + .ssekmsKeyId(response.sSEKMSKeyId()) + .bucketKeyEnabled(response.bucketKeyEnabled()) + .partsCount(response.partsCount()) + .tagCount(response.tagCount()) + .objectLockRetainUntilDate(response.objectLockRetainUntilDate()); + + if (response.serverSideEncryption() != null) { + builder.serverSideEncryption(response.serverSideEncryption().name()); + } + + if (response.storageClass() != null) { + builder.storageClass(response.storageClass().name()); + } + + if (response.requestCharged() != null) { + builder.requestCharged(response.requestCharged().name()); + } + + if (response.replicationStatus() != null) { + builder.replicationStatus(response.replicationStatus().name()); + } + + if (response.objectLockMode() != null) { + builder.objectLockMode(response.objectLockMode().name()); + } + + if (response.objectLockLegalHoldStatus() != null) { + builder.objectLockLegalHoldStatus(response.objectLockLegalHoldStatus().name()); + } + + return (GetObjectResponse) builder.responseMetadata(s3ResponseMetadata) + .sdkHttpResponse(sdkHttpResponse) + .build(); + + } + + public static com.amazonaws.s3.model.PutObjectRequest toCrtPutObjectRequest(PutObjectRequest sdkPutObject) { + com.amazonaws.s3.model.PutObjectRequest.Builder putObjectBuilder = + com.amazonaws.s3.model.PutObjectRequest.builder() + .contentLength(sdkPutObject.contentLength()) + .bucket(sdkPutObject.bucket()) + .key(sdkPutObject.key()) + .bucketKeyEnabled(sdkPutObject.bucketKeyEnabled()) + .cacheControl(sdkPutObject.cacheControl()) + .contentDisposition(sdkPutObject.contentDisposition()) + .contentEncoding(sdkPutObject.contentEncoding()) + .contentLanguage(sdkPutObject.contentLanguage()) + .contentMD5(sdkPutObject.contentMD5()) + .contentType(sdkPutObject.contentType()) + .expectedBucketOwner(sdkPutObject.expectedBucketOwner()) + .expires(sdkPutObject.expires()) + .grantFullControl(sdkPutObject.grantFullControl()) + .grantRead(sdkPutObject.grantRead()) + .grantReadACP(sdkPutObject.grantReadACP()) + .grantWriteACP(sdkPutObject.grantWriteACP()) + .metadata(sdkPutObject.metadata()) + .objectLockRetainUntilDate(sdkPutObject.objectLockRetainUntilDate()) + .sSECustomerAlgorithm(sdkPutObject.sseCustomerAlgorithm()) + .sSECustomerKey(sdkPutObject.sseCustomerKey()) + .sSECustomerKeyMD5(sdkPutObject.sseCustomerKeyMD5()) + .sSEKMSEncryptionContext(sdkPutObject.ssekmsEncryptionContext()) + .sSEKMSKeyId(sdkPutObject.ssekmsKeyId()) + .tagging(sdkPutObject.tagging()) + .websiteRedirectLocation(sdkPutObject.websiteRedirectLocation()); + + if (sdkPutObject.acl() != null) { + putObjectBuilder.aCL(ObjectCannedACL.fromValue(sdkPutObject.acl().name())); + } + + if (sdkPutObject.objectLockLegalHoldStatus() != null) { + putObjectBuilder.objectLockLegalHoldStatus(ObjectLockLegalHoldStatus.fromValue( + sdkPutObject.objectLockLegalHoldStatus().name())); + } + + if (sdkPutObject.objectLockMode() != null) { + putObjectBuilder.objectLockMode(ObjectLockMode.fromValue( + sdkPutObject.objectLockMode().name())); + } + + if (sdkPutObject.requestPayer() != null) { + putObjectBuilder.requestPayer(RequestPayer.fromValue(sdkPutObject.requestPayer().name())); + } + + if (sdkPutObject.serverSideEncryption() != null) { + putObjectBuilder.serverSideEncryption(ServerSideEncryption.fromValue( + sdkPutObject.serverSideEncryption().name())); + } + + if (sdkPutObject.storageClass() != null) { + putObjectBuilder.storageClass(StorageClass.fromValue( + sdkPutObject.storageClass().name())); + } + + processRequestOverrideConfiguration(sdkPutObject.overrideConfiguration().orElse(null), + putObjectBuilder::customQueryParameters); + + addCustomHeaders(sdkPutObject.overrideConfiguration().orElse(null), putObjectBuilder::customHeaders); + + return putObjectBuilder.build(); + } + + public static PutObjectResponse fromCrtPutObjectOutput(PutObjectOutput crtPutObjectOutput, + SdkHttpResponse sdkHttpResponse) { + S3ResponseMetadata s3ResponseMetadata = createS3ResponseMetadata(sdkHttpResponse); + PutObjectResponse.Builder builder = PutObjectResponse.builder() + .bucketKeyEnabled(crtPutObjectOutput.bucketKeyEnabled()) + .eTag(crtPutObjectOutput.eTag()) + .expiration(crtPutObjectOutput.expiration()) + .sseCustomerAlgorithm(crtPutObjectOutput.sSECustomerAlgorithm()) + .sseCustomerKeyMD5(crtPutObjectOutput.sSECustomerKeyMD5()) + .ssekmsEncryptionContext( + crtPutObjectOutput.sSEKMSEncryptionContext()) + .ssekmsKeyId(crtPutObjectOutput.sSEKMSKeyId()) + .versionId(crtPutObjectOutput.versionId()); + + if (crtPutObjectOutput.requestCharged() != null) { + builder.requestCharged(crtPutObjectOutput.requestCharged().name()); + } + + if (crtPutObjectOutput.serverSideEncryption() != null) { + builder.serverSideEncryption(crtPutObjectOutput.serverSideEncryption().name()); + } + + return (PutObjectResponse) builder.responseMetadata(s3ResponseMetadata) + .sdkHttpResponse(sdkHttpResponse) + .build(); + } + + private static S3ResponseMetadata createS3ResponseMetadata(SdkHttpResponse sdkHttpResponse) { + Map metadata = new HashMap<>(); + sdkHttpResponse.headers().forEach((key, value) -> metadata.put(key, value.get(0))); + return S3ResponseMetadata.create(DefaultAwsResponseMetadata.create(metadata)); + } + + private static void throwExceptionForUnsupportedConfigurations(AwsRequestOverrideConfiguration overrideConfiguration) { + if (!overrideConfiguration.metricPublishers().isEmpty()) { + throw new UnsupportedOperationException("Metric publishers are not supported"); + } + + if (overrideConfiguration.signer().isPresent()) { + throw new UnsupportedOperationException("signer is not supported"); + } + + if (!overrideConfiguration.apiNames().isEmpty()) { + throw new UnsupportedOperationException("apiNames is not supported"); + } + + if (overrideConfiguration.apiCallAttemptTimeout().isPresent()) { + throw new UnsupportedOperationException("apiCallAttemptTimeout is not supported"); + } + + if (overrideConfiguration.apiCallTimeout().isPresent()) { + throw new UnsupportedOperationException("apiCallTimeout is not supported"); + } + + if (overrideConfiguration.credentialsProvider().isPresent()) { + throw new UnsupportedOperationException("credentialsProvider is not supported"); + } + } + + private static void addRequestCustomHeaders(List crtHeaders, Map> headers) { + headers.forEach((key, value) -> { + value.stream().map(val -> new HttpHeader(key, val)).forEach(crtHeaders::add); + }); + } + + private static String encodedQueryString(Map> rawQueryParameters) { + return SdkHttpUtils.encodeAndFlattenQueryParameters(rawQueryParameters) + .map(value -> "?" + value) + .orElse(""); + } + + private static void processRequestOverrideConfiguration(AwsRequestOverrideConfiguration requestOverrideConfiguration, + Consumer queryParametersConsumer) { + if (requestOverrideConfiguration != null) { + throwExceptionForUnsupportedConfigurations(requestOverrideConfiguration); + + if (!requestOverrideConfiguration.rawQueryParameters().isEmpty()) { + String encodedQueryString = encodedQueryString(requestOverrideConfiguration.rawQueryParameters()); + queryParametersConsumer.accept(encodedQueryString); + } + } + } + + private static void addCustomHeaders(AwsRequestOverrideConfiguration requestOverrideConfiguration, + Consumer headersConsumer) { + + List crtHeaders = new ArrayList<>(); + crtHeaders.add(new HttpHeader(HEADER_USER_AGENT, USER_AGENT_STRING)); + + if (requestOverrideConfiguration != null && !requestOverrideConfiguration.headers().isEmpty()) { + addRequestCustomHeaders(crtHeaders, requestOverrideConfiguration.headers()); + } + + headersConsumer.accept(crtHeaders.toArray(new HttpHeader[0])); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfiguration.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfiguration.java new file mode 100644 index 000000000000..2bf435d699af --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfiguration.java @@ -0,0 +1,197 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + + +import static software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR; + +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; +import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider; +import software.amazon.awssdk.crt.io.ClientBootstrap; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.transfer.s3.SizeConstant; +import software.amazon.awssdk.utils.SdkAutoCloseable; +import software.amazon.awssdk.utils.ThreadFactoryBuilder; + +/** + * Internal client configuration resolver + */ +@SdkInternalApi +public class S3NativeClientConfiguration implements SdkAutoCloseable { + private static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * SizeConstant.MB; + private static final long DEFAULT_TARGET_THROUGHPUT_IN_GBPS = 5; + private final String signingRegion; + private final ClientBootstrap clientBootstrap; + private final CrtCredentialsProviderAdapter credentialProviderAdapter; + private final CredentialsProvider credentialsProvider; + private final long partSizeInBytes; + private final double targetThroughputInGbps; + private final int maxConcurrency; + private final Executor futureCompletionExecutor; + + public S3NativeClientConfiguration(Builder builder) { + this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() : + builder.signingRegion; + this.clientBootstrap = new ClientBootstrap(null, null); + + this.credentialProviderAdapter = + builder.credentialsProvider == null ? + new CrtCredentialsProviderAdapter(DefaultCredentialsProvider.create()) : + new CrtCredentialsProviderAdapter(builder.credentialsProvider); + + this.credentialsProvider = credentialProviderAdapter.crtCredentials(); + + this.partSizeInBytes = builder.partSizeInBytes == null ? DEFAULT_PART_SIZE_IN_BYTES : + builder.partSizeInBytes; + this.targetThroughputInGbps = builder.targetThroughputInGbps == null ? + DEFAULT_TARGET_THROUGHPUT_IN_GBPS : builder.targetThroughputInGbps; + + // Using 0 so that CRT will calculate it based on targetThroughputGbps + this.maxConcurrency = builder.maxConcurrency == null ? 0 : builder.maxConcurrency; + + this.futureCompletionExecutor = resolveAsyncFutureCompletionExecutor(builder.asynConfiguration); + } + + public static Builder builder() { + return new Builder(); + } + + public String signingRegion() { + return signingRegion; + } + + public ClientBootstrap clientBootstrap() { + return clientBootstrap; + } + + public CredentialsProvider credentialsProvider() { + return credentialsProvider; + } + + public long partSizeBytes() { + return partSizeInBytes; + } + + public double targetThroughputInGbps() { + return targetThroughputInGbps; + } + + public int maxConcurrency() { + return maxConcurrency; + } + + public Executor futureCompletionExecutor() { + return futureCompletionExecutor; + } + + /** + * Finalize which async executor service will be used for the created client. The default async executor + * service has at least 8 core threads and can scale up to at least 64 threads when needed depending + * on the number of processors available. + * + * This uses the same default executor from SdkDefaultClientBuilder#resolveAsyncFutureCompletionExecutor. + * Make sure you update that method if you update the defaults here. + */ + private Executor resolveAsyncFutureCompletionExecutor(ClientAsyncConfiguration config) { + Supplier defaultExecutor = () -> { + int processors = Runtime.getRuntime().availableProcessors(); + int corePoolSize = Math.max(8, processors); + int maxPoolSize = Math.max(64, processors * 2); + ThreadPoolExecutor executor = new ThreadPoolExecutor(corePoolSize, maxPoolSize, + 10, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1_000), + new ThreadFactoryBuilder() + .threadNamePrefix("sdk-async-response").build()); + // Allow idle core threads to time out + executor.allowCoreThreadTimeOut(true); + return executor; + }; + + return Optional.ofNullable(config) + .map(c -> c.advancedOption(FUTURE_COMPLETION_EXECUTOR)) + .orElseGet(defaultExecutor); + } + + @Override + public void close() { + clientBootstrap.close(); + credentialProviderAdapter.close(); + shutdownIfExecutorService(futureCompletionExecutor); + } + + private void shutdownIfExecutorService(Object object) { + if (object instanceof ExecutorService) { + ExecutorService executor = (ExecutorService) object; + executor.shutdown(); + } + } + + public static final class Builder { + private String signingRegion; + private AwsCredentialsProvider credentialsProvider; + private Long partSizeInBytes; + private Double targetThroughputInGbps; + private Integer maxConcurrency; + private ClientAsyncConfiguration asynConfiguration; + + private Builder() { + } + + public Builder signingRegion(String signingRegion) { + this.signingRegion = signingRegion; + return this; + } + + public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + public Builder partSizeInBytes(Long partSizeInBytes) { + this.partSizeInBytes = partSizeInBytes; + return this; + } + + public Builder targetThroughputInGbps(Double targetThroughputInGbps) { + this.targetThroughputInGbps = targetThroughputInGbps; + return this; + } + + public Builder maxConcurrency(Integer maxConcurrency) { + this.maxConcurrency = maxConcurrency; + return this; + } + + public Builder asyncConfiguration(ClientAsyncConfiguration asyncConfiguration) { + this.asynConfiguration = asyncConfiguration; + return this; + } + + public S3NativeClientConfiguration build() { + return new S3NativeClientConfiguration(this); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java new file mode 100644 index 000000000000..db7583cb559f --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/DownloadRequestTest.java @@ -0,0 +1,79 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Paths; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; + +public class DownloadRequestTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void noGetObjectRequest_throws() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("getObjectRequest"); + + DownloadRequest.builder() + .destination(Paths.get(".")) + .build(); + } + + @Test + public void pathMissing_throws() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("destination"); + + DownloadRequest.builder() + .getObjectRequest(b -> b.bucket("bucket").key("key")) + .build(); + } + + @Test + public void equals_hashcode() { + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket("bucket") + .key("key") + .build(); + + DownloadRequest request1 = DownloadRequest.builder() + .getObjectRequest(b -> b.bucket("bucket").key("key")) + .destination(Paths.get(".")) + .build(); + + DownloadRequest request2 = DownloadRequest.builder() + .getObjectRequest(getObjectRequest) + .destination(Paths.get(".")) + .build(); + + DownloadRequest request3 = DownloadRequest.builder() + .getObjectRequest(b -> b.bucket("bucket1").key("key1")) + .destination(Paths.get(".")) + .build(); + + assertThat(request1).isEqualTo(request2); + assertThat(request1.hashCode()).isEqualTo(request2.hashCode()); + + assertThat(request1.hashCode()).isNotEqualTo(request3.hashCode()); + assertThat(request1).isNotEqualTo(request3); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java new file mode 100644 index 000000000000..9788da748654 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/S3ClientConfigurationTest.java @@ -0,0 +1,132 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static software.amazon.awssdk.transfer.s3.SizeConstant.MB; + +import org.junit.Test; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.regions.Region; + +public class S3ClientConfigurationTest { + + @Test + public void nonPositiveMinimumPartSizeInBytes_shouldThrowException() { + assertThatThrownBy(() -> S3ClientConfiguration.builder() + .minimumPartSizeInBytes(-10L) + .build()) + .hasMessageContaining("must be positive"); + assertThatThrownBy(() -> S3ClientConfiguration.builder() + .minimumPartSizeInBytes(0L) + .build()) + .hasMessageContaining("must be positive"); + + } + + @Test + public void nonPositiveTargetThroughput_shouldThrowException() { + assertThatThrownBy(() -> S3ClientConfiguration.builder() + .targetThroughputInGbps(-10.0) + .build()) + .hasMessageContaining("must be positive"); + assertThatThrownBy(() -> S3ClientConfiguration.builder() + .targetThroughputInGbps(0.0) + .build()) + .hasMessageContaining("must be positive"); + } + + @Test + public void nonPositiveMaxConcurrency_shouldThrowException() { + assertThatThrownBy(() -> S3ClientConfiguration.builder() + .maxConcurrency(-10) + .build()) + .hasMessageContaining("must be positive"); + assertThatThrownBy(() -> S3ClientConfiguration.builder() + .maxConcurrency(0) + .build()) + .hasMessageContaining("must be positive"); + } + + @Test + public void build_allProperties() { + AwsCredentialsProvider credentials = () -> AwsBasicCredentials.create("test" + , "test"); + S3ClientConfiguration configuration = S3ClientConfiguration.builder() + .credentialsProvider(credentials) + .maxConcurrency(100) + .targetThroughputInGbps(10.0) + .region(Region.US_WEST_2) + .minimumPartSizeInBytes(5 * MB) + .build(); + + assertThat(configuration.credentialsProvider()).contains(credentials); + assertThat(configuration.maxConcurrency()).contains(100); + assertThat(configuration.region()).contains(Region.US_WEST_2); + assertThat(configuration.targetThroughputInGbps()).contains(10.0); + assertThat(configuration.minimumPartSizeInBytes()).contains(5 * MB); + } + + @Test + public void build_emptyBuilder() { + S3ClientConfiguration configuration = S3ClientConfiguration.builder() + .build(); + + assertThat(configuration.credentialsProvider()).isEmpty(); + assertThat(configuration.maxConcurrency()).isEmpty(); + assertThat(configuration.region()).isEmpty(); + assertThat(configuration.targetThroughputInGbps()).isEmpty(); + assertThat(configuration.minimumPartSizeInBytes()).isEmpty(); + } + + @Test + public void equalsHashCode() { + AwsCredentialsProvider credentials = () -> AwsBasicCredentials.create("test" + , "test"); + S3ClientConfiguration configuration1 = S3ClientConfiguration.builder() + .credentialsProvider(credentials) + .maxConcurrency(100) + .targetThroughputInGbps(10.0) + .region(Region.US_WEST_2) + .minimumPartSizeInBytes(5 * MB) + .build(); + + S3ClientConfiguration configuration2 = S3ClientConfiguration.builder() + .credentialsProvider(credentials) + .maxConcurrency(100) + .targetThroughputInGbps(10.0) + .region(Region.US_WEST_2) + .minimumPartSizeInBytes(5 * MB) + .build(); + + S3ClientConfiguration configuration3 = configuration1.toBuilder() + .credentialsProvider(AnonymousCredentialsProvider.create()) + .maxConcurrency(50) + .targetThroughputInGbps(1.0) + .asyncConfiguration(c -> c.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, + Runnable::run)) + .build(); + + assertThat(configuration1).isEqualTo(configuration2); + assertThat(configuration1.hashCode()).isEqualTo(configuration2.hashCode()); + assertThat(configuration1).isNotEqualTo(configuration3); + assertThat(configuration1.hashCode()).isNotEqualTo(configuration3.hashCode()); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java new file mode 100644 index 000000000000..b1ee981a9e5c --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/UploadRequestTest.java @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Paths; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +public class UploadRequestTest { + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void upload_noRequestParamsProvided_throws() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("putObjectRequest"); + + UploadRequest.builder() + .source(Paths.get(".")) + .build(); + } + + @Test + public void pathMissing_shouldThrow() { + thrown.expect(NullPointerException.class); + thrown.expectMessage("source"); + UploadRequest.builder() + .putObjectRequest(PutObjectRequest.builder().build()) + .build(); + } + + @Test + public void equals_hashcode() { + PutObjectRequest getObjectRequest = PutObjectRequest.builder() + .bucket("bucket") + .key("key") + .build(); + + UploadRequest request1 = UploadRequest.builder() + .putObjectRequest(b -> b.bucket("bucket").key("key")) + .source(Paths.get(".")) + .build(); + + UploadRequest request2 = UploadRequest.builder() + .putObjectRequest(getObjectRequest) + .source(Paths.get(".")) + .build(); + + UploadRequest request3 = UploadRequest.builder() + .putObjectRequest(b -> b.bucket("bucket1").key("key1")) + .source(Paths.get(".")) + .build(); + + assertThat(request1).isEqualTo(request2); + assertThat(request1.hashCode()).isEqualTo(request2.hashCode()); + + assertThat(request1.hashCode()).isNotEqualTo(request3.hashCode()); + assertThat(request1).isNotEqualTo(request3); + } + +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialProviderAdapterTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialProviderAdapterTest.java new file mode 100644 index 000000000000..b3769c02b383 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtCredentialProviderAdapterTest.java @@ -0,0 +1,93 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import org.junit.Test; +import org.mockito.Mockito; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.HttpCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.crt.auth.credentials.Credentials; +import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider; + +public class CrtCredentialProviderAdapterTest { + + @Test + public void crtCredentials_withSession_shouldConvert() { + AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider + .create(AwsSessionCredentials.create("foo", "bar", "session")); + + CredentialsProvider crtCredentialsProvider = new CrtCredentialsProviderAdapter(awsCredentialsProvider) + .crtCredentials(); + + Credentials credentials = crtCredentialsProvider.getCredentials().join(); + + assertThat(credentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8)); + assertThat(credentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8)); + assertThat(credentials.getSessionToken()).isEqualTo("session".getBytes(StandardCharsets.UTF_8)); + } + + @Test + public void crtCredentials_withoutSession_shouldConvert() { + AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider + .create(AwsBasicCredentials.create("foo", "bar")); + + CredentialsProvider crtCredentialsProvider = new CrtCredentialsProviderAdapter(awsCredentialsProvider) + .crtCredentials(); + + Credentials credentials = crtCredentialsProvider.getCredentials().join(); + + assertThat(credentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8)); + assertThat(credentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8)); + assertThat(credentials.getSessionToken()).isNull(); + } + + @Test + public void crtCredentials_provideAwsCredentials_shouldInvokeResolveAndClose() { + HttpCredentialsProvider awsCredentialsProvider = Mockito.mock(HttpCredentialsProvider.class); + AwsCredentials credentials = new AwsCredentials() { + @Override + public String accessKeyId() { + return "foo"; + } + + @Override + public String secretAccessKey() { + return "bar"; + } + }; + when(awsCredentialsProvider.resolveCredentials()).thenReturn(credentials); + + CrtCredentialsProviderAdapter adapter = new CrtCredentialsProviderAdapter(awsCredentialsProvider); + CredentialsProvider crtCredentialsProvider = adapter.crtCredentials(); + + Credentials crtCredentials = crtCredentialsProvider.getCredentials().join(); + assertThat(crtCredentials.getAccessKeyId()).isEqualTo("foo".getBytes(StandardCharsets.UTF_8)); + assertThat(crtCredentials.getSecretAccessKey()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8)); + verify(awsCredentialsProvider).resolveCredentials(); + + adapter.close(); + verify(awsCredentialsProvider).close(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandlerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandlerTest.java new file mode 100644 index 000000000000..9d23be5c9781 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandlerTest.java @@ -0,0 +1,111 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.exception.SdkServiceException; +import software.amazon.awssdk.crt.CrtRuntimeException; +import software.amazon.awssdk.crt.s3.CrtS3RuntimeException; +import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class CrtErrorHandlerTest { + + @Mock + private CrtS3RuntimeException mockCrtS3RuntimeException; + + @Test + public void crtS3ExceptionAreTransformed(){ + CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); + when(mockCrtS3RuntimeException.getAwsErrorCode()).thenReturn("BucketAlreadyExists"); + when(mockCrtS3RuntimeException.getAwsErrorMessage()).thenReturn("Bucket Already Exists"); + when(mockCrtS3RuntimeException.getStatusCode()).thenReturn(404); + Exception transformException = crtErrorHandler.transformException(mockCrtS3RuntimeException); + Assertions.assertThat(transformException).isInstanceOf(BucketAlreadyExistsException.class); + Assertions.assertThat(transformException.getMessage()).contains("Bucket Already Exists"); + } + + @Test + public void nonCrtS3ExceptionAreNotTransformed(){ + CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); + Exception transformException = crtErrorHandler.transformException(new CrtRuntimeException("AWS_ERROR")); + Assertions.assertThat(transformException).isInstanceOf(SdkClientException.class); + } + + + @Test + public void crtS3ExceptionAreTransformedWhenExceptionIsInCause(){ + CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); + when(mockCrtS3RuntimeException.getAwsErrorCode()).thenReturn("InvalidObjectState"); + when(mockCrtS3RuntimeException.getAwsErrorMessage()).thenReturn("Invalid Object State"); + when(mockCrtS3RuntimeException.getStatusCode()).thenReturn(404); + final Exception transformException = crtErrorHandler.transformException(new Exception("Some Exception", mockCrtS3RuntimeException)); + + System.out.println("transformException " +transformException); + + Assertions.assertThat(transformException).isInstanceOf(InvalidObjectStateException.class); + Assertions.assertThat(transformException.getMessage()).contains("Invalid Object State"); + Assertions.assertThat(transformException.getCause()).isInstanceOf(CrtS3RuntimeException.class); + } + + @Test + public void nonCrtS3ExceptionAreNotTransformedWhenExceptionIsInCause(){ + CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); + final Exception crtRuntimeException = new Exception("Some Exception", new CrtRuntimeException("AWS_ERROR")); + Exception transformException = crtErrorHandler.transformException( + crtRuntimeException); + Assertions.assertThat(transformException).isNotInstanceOf(CrtRuntimeException.class); + Assertions.assertThat(transformException).isInstanceOf(SdkClientException.class); + Assertions.assertThat(transformException.getMessage()).isEqualTo("Some Exception"); + Assertions.assertThat(transformException.getCause()).isEqualTo(crtRuntimeException); + } + + @Test + public void crtS3ExceptionWithErrorCodeNodeNotInS3Model() { + CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); + when(mockCrtS3RuntimeException.getAwsErrorCode()).thenReturn("NewS3ExceptionFromCrt"); + when(mockCrtS3RuntimeException.getAwsErrorMessage()).thenReturn("New S3 Exception From Crt"); + when(mockCrtS3RuntimeException.getStatusCode()).thenReturn(404); + Exception transformException = crtErrorHandler.transformException(mockCrtS3RuntimeException); + Assertions.assertThat(transformException).isInstanceOf(SdkServiceException.class); + Assertions.assertThat(transformException.getCause()).isEqualTo(mockCrtS3RuntimeException); + Assertions.assertThat(transformException.getMessage()).isEqualTo(mockCrtS3RuntimeException.getMessage()); + Assertions.assertThat(((SdkServiceException)transformException).statusCode()) + .isEqualTo(mockCrtS3RuntimeException.getStatusCode()); + } + + @Test + public void crtS3ExceptionInCauseWithErrorCodeNodeNotInS3Model() { + CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); + when(mockCrtS3RuntimeException.getAwsErrorCode()).thenReturn("NewS3ExceptionFromCrt"); + when(mockCrtS3RuntimeException.getAwsErrorMessage()).thenReturn("New S3 Exception From Crt"); + when(mockCrtS3RuntimeException.getStatusCode()).thenReturn(404); + final Exception crtRuntimeException = new Exception(mockCrtS3RuntimeException); + Exception transformException = crtErrorHandler.transformException(crtRuntimeException); + Assertions.assertThat(transformException).isInstanceOf(SdkServiceException.class); + Assertions.assertThat(transformException.getCause()).isEqualTo(mockCrtS3RuntimeException); + Assertions.assertThat(transformException.getMessage()).isEqualTo(mockCrtS3RuntimeException.getMessage()); + Assertions.assertThat(((SdkServiceException) transformException).statusCode()).isEqualTo(404); + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapterTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapterTest.java new file mode 100644 index 000000000000..87d2a1181252 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapterTest.java @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; + +import com.amazonaws.s3.model.GetObjectOutput; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +@RunWith(MockitoJUnitRunner.class) +public class CrtResponseDataConsumerAdapterTest { + private CrtResponseDataConsumerAdapter adapter; + + @Mock + private S3CrtDataPublisher publisher; + + @Mock + private AsyncResponseTransformer transformer; + + @Before + public void setup() { + ResponseHeadersHandler handler = new ResponseHeadersHandler(); + adapter = new CrtResponseDataConsumerAdapter<>(transformer, publisher, handler); + } + + @Test + public void onResponse_noSdkHttpResponse_shouldCreateEmptySdkHttpResponse() { + adapter.onResponse(GetObjectOutput.builder().build()); + ArgumentCaptor captor = ArgumentCaptor.forClass(GetObjectResponse.class); + verify(transformer).onResponse(captor.capture()); + assertThat(captor.getValue().responseMetadata().requestId()).isEqualTo("UNKNOWN"); + assertThat(captor.getValue().sdkHttpResponse()).isNotNull(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClientTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClientTest.java new file mode 100644 index 000000000000..9e56594d68c9 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClientTest.java @@ -0,0 +1,161 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazonaws.s3.RequestDataSupplier; +import com.amazonaws.s3.ResponseDataConsumer; +import com.amazonaws.s3.S3NativeClient; +import com.amazonaws.s3.model.GetObjectOutput; +import com.amazonaws.s3.model.GetObjectRequest; +import com.amazonaws.s3.model.PutObjectOutput; +import com.amazonaws.s3.model.PutObjectRequest; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultS3CrtAsyncClientTest { + @Mock + private S3NativeClient mockS3NativeClient; + + @Mock + private S3NativeClientConfiguration mockConfiguration; + + private S3CrtAsyncClient s3CrtAsyncClient; + + private static ExecutorService executor; + + @BeforeClass + public static void setUp() { + executor = Executors.newSingleThreadExecutor(); + } + + @Before + public void methodSetup() { + s3CrtAsyncClient = new DefaultS3CrtAsyncClient(mockConfiguration, + mockS3NativeClient); + when(mockConfiguration.futureCompletionExecutor()).thenReturn(executor); + } + + @AfterClass + public static void cleanUp() { + executor.shutdown(); + } + + @Test + public void getObject_cancels_shouldForwardCancellation() { + CompletableFuture crtFuture = new CompletableFuture<>(); + when(mockS3NativeClient.getObject(any(GetObjectRequest.class), + any(ResponseDataConsumer.class))) + .thenReturn(crtFuture); + + CompletableFuture> future = + s3CrtAsyncClient.getObject(b -> b.bucket("bucket").key("key"), + AsyncResponseTransformer.toBytes()); + + future.cancel(true); + assertThat(crtFuture).isCancelled(); + } + + @Test + public void putObject_cancels_shouldForwardCancellation() { + CompletableFuture crtFuture = new CompletableFuture<>(); + when(mockS3NativeClient.putObject(any(PutObjectRequest.class), + any(RequestDataSupplier.class))) + .thenReturn(crtFuture); + + CompletableFuture future = + s3CrtAsyncClient.putObject(b -> b.bucket("bucket").key("key"), + AsyncRequestBody.empty()); + + future.cancel(true); + assertThat(crtFuture).isCancelled(); + } + + @Test + public void putObject_crtFutureCompletedExceptionally_shouldFail() { + RuntimeException runtimeException = new RuntimeException("test"); + CompletableFuture crtFuture = new CompletableFuture<>(); + crtFuture.completeExceptionally(runtimeException); + when(mockS3NativeClient.putObject(any(PutObjectRequest.class), + any(RequestDataSupplier.class))) + .thenReturn(crtFuture); + + CompletableFuture future = + s3CrtAsyncClient.putObject(b -> b.bucket("bucket").key("key"), + AsyncRequestBody.empty()); + + assertThatThrownBy(() -> future.join()).hasCause(SdkClientException + .create("java.lang.RuntimeException: test", runtimeException)); + } + + @Test + public void getObject_crtFutureCompletedExceptionally_shouldFail() { + RuntimeException runtimeException = new RuntimeException("test"); + CompletableFuture crtFuture = new CompletableFuture<>(); + crtFuture.completeExceptionally(runtimeException); + when(mockS3NativeClient.getObject(any(GetObjectRequest.class), + any(ResponseDataConsumer.class))) + .thenReturn(crtFuture); + + CompletableFuture> future = + s3CrtAsyncClient.getObject(b -> b.bucket("bucket").key("key"), + AsyncResponseTransformer.toBytes()); + + assertThatThrownBy(() -> future.join()).hasCause(SdkClientException.create("test", runtimeException)); + } + + @Test + public void putObject_crtFutureCompletedSuccessfully_shouldSucceed() { + CompletableFuture crtFuture = new CompletableFuture<>(); + crtFuture.complete(PutObjectOutput.builder().build()); + when(mockS3NativeClient.putObject(any(PutObjectRequest.class), + any(RequestDataSupplier.class))) + .thenReturn(crtFuture); + + CompletableFuture future = + s3CrtAsyncClient.putObject(b -> b.bucket("bucket").key("key"), + AsyncRequestBody.empty()); + + assertThat(future.join().sdkHttpResponse().statusText()).isEmpty(); + } + + @Test + public void closeS3Client_shouldCloseUnderlyingResources() { + s3CrtAsyncClient.close(); + verify(mockS3NativeClient).close(); + verify(mockConfiguration).close(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTckTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTckTest.java new file mode 100644 index 000000000000..9f202bde9ad1 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTckTest.java @@ -0,0 +1,75 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import org.reactivestreams.tck.SubscriberWhiteboxVerification; +import org.reactivestreams.tck.TestEnvironment; + +public class RequestDataSupplierAdapterTckTest extends SubscriberWhiteboxVerification { + private static final byte[] CONTENT = new byte[16]; + + protected RequestDataSupplierAdapterTckTest() { + super(new TestEnvironment()); + } + + @Override + public Subscriber createSubscriber(WhiteboxSubscriberProbe whiteboxSubscriberProbe) { + return new RequestDataSupplierAdapter.SubscriberImpl((s) -> {}, new ArrayDeque<>()) { + @Override + public void onSubscribe(Subscription subscription) { + super.onSubscribe(subscription); + whiteboxSubscriberProbe.registerOnSubscribe(new SubscriberPuppet() { + @Override + public void triggerRequest(long l) { + subscription.request(l); + } + + @Override + public void signalCancel() { + subscription.cancel(); + } + }); + } + + @Override + public void onNext(ByteBuffer bb) { + super.onNext(bb); + whiteboxSubscriberProbe.registerOnNext(bb); + } + + @Override + public void onError(Throwable t) { + super.onError(t); + whiteboxSubscriberProbe.registerOnError(t); + } + + @Override + public void onComplete() { + super.onComplete(); + whiteboxSubscriberProbe.registerOnComplete(); + } + }; + } + + @Override + public ByteBuffer createElement(int i) { + return ByteBuffer.wrap(CONTENT); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTest.java new file mode 100644 index 000000000000..b01cada9e1c6 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTest.java @@ -0,0 +1,217 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import io.reactivex.Flowable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.Test; +import org.reactivestreams.Publisher; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.crt.CrtRuntimeException; + +public class RequestDataSupplierAdapterTest { + + @Test + public void getRequestData_fillsInputBuffer_publisherBuffersAreSmaller() { + int inputBufferSize = 16; + + List data = Stream.generate(() -> (byte) 42) + .limit(inputBufferSize) + .map(b -> { + ByteBuffer bb = ByteBuffer.allocate(1); + bb.put(b); + bb.flip(); + return bb; + }) + .collect(Collectors.toList()); + + AsyncRequestBody requestBody = AsyncRequestBody.fromPublisher(Flowable.fromIterable(data)); + + RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + + ByteBuffer inputBuffer = ByteBuffer.allocate(inputBufferSize); + adapter.getRequestBytes(inputBuffer); + + assertThat(inputBuffer.remaining()).isEqualTo(0); + } + + @Test + public void getRequestData_fillsInputBuffer_publisherBuffersAreLarger() { + int bodySize = 16; + + ByteBuffer data = ByteBuffer.allocate(bodySize); + data.put(new byte[bodySize]); + data.flip(); + + AsyncRequestBody requestBody = AsyncRequestBody.fromPublisher(Flowable.just(data)); + + RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + + ByteBuffer inputBuffer = ByteBuffer.allocate(1); + + for (int i = 0; i < bodySize; ++i) { + adapter.getRequestBytes(inputBuffer); + assertThat(inputBuffer.remaining()).isEqualTo(0); + inputBuffer.flip(); + } + } + + @Test + public void getRequestData_publisherThrows_surfacesException() { + Publisher errorPublisher = Flowable.error(new RuntimeException("Something wrong happened")); + + AsyncRequestBody requestBody = AsyncRequestBody.fromPublisher(errorPublisher); + RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + + assertThatThrownBy(() -> adapter.getRequestBytes(ByteBuffer.allocate(16))) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Something wrong happened"); + } + + @Test + public void getRequestData_publisherThrows_wrapsExceptionIfNotRuntimeException() { + Publisher errorPublisher = Flowable.error(new IOException("Some I/O error happened")); + + AsyncRequestBody requestBody = AsyncRequestBody.fromPublisher(errorPublisher); + RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + + assertThatThrownBy(() -> adapter.getRequestBytes(ByteBuffer.allocate(16))) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(IOException.class); + } + + @Test + public void resetMidStream_discardsBufferedData() { + long requestSize = RequestDataSupplierAdapter.DEFAULT_REQUEST_SIZE; + int inputBufferSize = 16; + + Publisher requestBody = new Publisher() { + private byte value = 0; + + @Override + public void subscribe(Subscriber subscriber) { + byte byteVal = value++; + + List dataList = Stream.generate(() -> { + byte[] data = new byte[inputBufferSize]; + Arrays.fill(data, byteVal); + return ByteBuffer.wrap(data); + }) + .limit(requestSize) + .collect(Collectors.toList()); + + Flowable realPublisher = Flowable.fromIterable(dataList); + + realPublisher.subscribe(subscriber); + } + }; + + RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + + long resetAfter = requestSize / 2; + + ByteBuffer inputBuffer = ByteBuffer.allocate(inputBufferSize); + + for (long l = 0; l < resetAfter; ++l) { + adapter.getRequestBytes(inputBuffer); + inputBuffer.flip(); + } + + adapter.resetPosition(); + + byte[] expectedBufferContent = new byte[inputBufferSize]; + Arrays.fill(expectedBufferContent, (byte) 1); + + byte[] readBuffer = new byte[inputBufferSize]; + for (int l = 0; l < requestSize; ++l) { + adapter.getRequestBytes(inputBuffer); + // flip for reading + inputBuffer.flip(); + inputBuffer.get(readBuffer); + + // flip for writing + inputBuffer.flip(); + + assertThat(readBuffer).isEqualTo(expectedBufferContent); + } + } + + @Test + public void onException_cancelsSubscription() { + Subscription subscription = mock(Subscription.class); + + AsyncRequestBody requestBody = new AsyncRequestBody() { + @Override + public Optional contentLength() { + return Optional.empty(); + } + + @Override + public void subscribe(Subscriber subscriber) { + subscriber.onSubscribe(subscription); + } + }; + + RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + + // getRequestBytes() triggers a subscribe() on the publisher + adapter.getRequestBytes(ByteBuffer.allocate(0)); + + adapter.onException(new CrtRuntimeException("error")); + + verify(subscription).cancel(); + } + + @Test + public void onFinished_cancelsSubscription() { + Subscription subscription = mock(Subscription.class); + + AsyncRequestBody requestBody = new AsyncRequestBody() { + @Override + public Optional contentLength() { + return Optional.empty(); + } + + @Override + public void subscribe(Subscriber subscriber) { + subscriber.onSubscribe(subscription); + } + }; + + RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + + // getRequestBytes() triggers a subscribe() on the publisher + adapter.getRequestBytes(ByteBuffer.allocate(0)); + + adapter.onFinished(); + + verify(subscription).cancel(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandlerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandlerTest.java new file mode 100644 index 000000000000..847517181623 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandlerTest.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import software.amazon.awssdk.crt.http.HttpHeader; + +public class ResponseHeadersHandlerTest { + private ResponseHeadersHandler handler; + + @Before + public void setUp() { + handler = new ResponseHeadersHandler(); + } + + @Test + public void onResponseHeaders_shouldCreateSdkHttpResponse() { + HttpHeader[] headers = new HttpHeader[1]; + headers[0] = new HttpHeader("foo", "bar"); + + handler.onResponseHeaders(400, headers); + assertThat(handler.sdkHttpResponseFuture()).isCompleted(); + Map> actualHeaders + = handler.sdkHttpResponseFuture().join().headers(); + assertThat(actualHeaders).hasSize(1); + assertThat(actualHeaders.get("foo")).containsExactlyInAnyOrder("bar"); + } + + @Test + public void responseNotReady() { + assertThat(handler.sdkHttpResponseFuture()).isNotCompleted(); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisherTckTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisherTckTest.java new file mode 100644 index 000000000000..acd2009df3b1 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisherTckTest.java @@ -0,0 +1,61 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.nio.ByteBuffer; +import org.apache.commons.lang3.RandomUtils; +import org.reactivestreams.Publisher; +import org.reactivestreams.tck.TestEnvironment; +import software.amazon.awssdk.core.internal.async.FileAsyncRequestBody; + +/** + * TCK verification test for {@link FileAsyncRequestBody}. + */ +public class S3CrtDataPublisherTckTest extends org.reactivestreams.tck.PublisherVerification { + + public S3CrtDataPublisherTckTest() { + super(new TestEnvironment()); + } + + @Override + public Publisher createPublisher(long elements) { + S3CrtDataPublisher s3CrtDataPublisher = new S3CrtDataPublisher(); + + for (long i = 0; i < elements; i++) { + s3CrtDataPublisher.deliverData(ByteBuffer.wrap(RandomUtils.nextBytes(20))); + } + + s3CrtDataPublisher.notifyStreamingFinished(); + + return s3CrtDataPublisher; + } + + @Override + public long maxElementsFromPublisher() { + return 1024; + } + + @Override + public Publisher createFailedPublisher() { + S3CrtDataPublisher s3CrtDataPublisher = new S3CrtDataPublisher(); + + s3CrtDataPublisher.deliverData(ByteBuffer.wrap(RandomUtils.nextBytes(20))); + + s3CrtDataPublisher.notifyError(new RuntimeException("error")); + + return s3CrtDataPublisher; + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisherTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisherTest.java new file mode 100644 index 000000000000..28d2ac3ff1d1 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtDataPublisherTest.java @@ -0,0 +1,222 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.commons.lang3.RandomUtils; +import org.junit.Before; +import org.junit.Test; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +public class S3CrtDataPublisherTest { + + private S3CrtDataPublisher dataPublisher; + + @Before + public void setup() { + dataPublisher = new S3CrtDataPublisher(); + } + + @Test + public void publisherFinishesSuccessfully_shouldInvokeOnComplete() throws InterruptedException { + AtomicBoolean errorOccurred = new AtomicBoolean(false); + CountDownLatch countDownLatch = new CountDownLatch(1); + Queue events = new ConcurrentLinkedQueue<>(); + int numOfData = 3; + dataPublisher.subscribe(new Subscriber() { + private Subscription subscription; + @Override + public void onSubscribe(Subscription s) { + this.subscription = s; + s.request(1); + } + + @Override + public void onNext(ByteBuffer byteBuffer) { + events.add(byteBuffer); + subscription.request(Long.MAX_VALUE); + } + + @Override + public void onError(Throwable t) { + errorOccurred.set(true); + } + + @Override + public void onComplete() { + countDownLatch.countDown(); + } + }); + + for (int i = 0; i < numOfData; i++) { + dataPublisher.deliverData(ByteBuffer.wrap(RandomUtils.nextBytes(20))); + } + + dataPublisher.notifyStreamingFinished(); + + countDownLatch.await(5, TimeUnit.SECONDS); + assertThat(errorOccurred).isFalse(); + assertThat(events.size()).isEqualTo(numOfData); + } + + @Test + public void publisherHasOneByteBuffer_subscriberRequestOnce_shouldInvokeComplete() throws InterruptedException { + AtomicBoolean errorOccurred = new AtomicBoolean(false); + CountDownLatch countDownLatch = new CountDownLatch(1); + Queue events = new ConcurrentLinkedQueue<>(); + int numOfData = 1; + dataPublisher.subscribe(new Subscriber() { + private Subscription subscription; + @Override + public void onSubscribe(Subscription s) { + this.subscription = s; + s.request(1); + } + + @Override + public void onNext(ByteBuffer byteBuffer) { + events.add(byteBuffer); + } + + @Override + public void onError(Throwable t) { + errorOccurred.set(true); + } + + @Override + public void onComplete() { + countDownLatch.countDown(); + } + }); + + for (int i = 0; i < numOfData; i++) { + dataPublisher.deliverData(ByteBuffer.wrap(RandomUtils.nextBytes(20))); + } + + dataPublisher.notifyStreamingFinished(); + + countDownLatch.await(5, TimeUnit.SECONDS); + assertThat(countDownLatch.getCount()).isEqualTo(0); + assertThat(errorOccurred).isFalse(); + assertThat(events.size()).isEqualTo(numOfData); + } + + @Test + public void publisherThrowsError_shouldInvokeOnError() throws InterruptedException { + AtomicBoolean onCompleteCalled = new AtomicBoolean(false); + CountDownLatch countDownLatch = new CountDownLatch(1); + Queue events = new ConcurrentLinkedQueue<>(); + int numOfData = 3; + dataPublisher.subscribe(new Subscriber() { + private Subscription subscription; + @Override + public void onSubscribe(Subscription s) { + this.subscription = s; + s.request(1); + } + + @Override + public void onNext(ByteBuffer byteBuffer) { + events.add(byteBuffer); + subscription.request(Long.MAX_VALUE); + } + + @Override + public void onError(Throwable t) { + countDownLatch.countDown(); + } + + @Override + public void onComplete() { + onCompleteCalled.set(true); + } + }); + + for (int i = 0; i < numOfData; i++) { + dataPublisher.deliverData(ByteBuffer.wrap(RandomUtils.nextBytes(20))); + } + + dataPublisher.notifyError(new RuntimeException("test")); + + countDownLatch.await(5, TimeUnit.SECONDS); + assertThat(onCompleteCalled).isFalse(); + assertThat(events.size()).isEqualTo(numOfData); + } + + @Test + public void subscriberCancels_shouldNotInvokeTerminalMethods() { + AtomicBoolean onCompleteCalled = new AtomicBoolean(false); + AtomicBoolean errorOccurred = new AtomicBoolean(false); + + Queue events = new ConcurrentLinkedQueue<>(); + List> futures = new ArrayList<>(); + int numOfData = 3; + for (int i = 0; i < numOfData; i++) { + futures.add( + CompletableFuture.runAsync(() -> dataPublisher.deliverData(ByteBuffer.wrap(RandomUtils.nextBytes(20))))); + } + + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).whenComplete((r, t) -> { + CompletableFuture.runAsync(() -> dataPublisher.notifyStreamingFinished()); + }); + + dataPublisher.subscribe(new Subscriber() { + private Subscription subscription; + @Override + public void onSubscribe(Subscription s) { + this.subscription = s; + s.request(Long.MAX_VALUE); + } + + @Override + public void onNext(ByteBuffer byteBuffer) { + events.add(byteBuffer); + subscription.cancel(); + } + + @Override + public void onError(Throwable t) { + errorOccurred.set(true); + } + + @Override + public void onComplete() { + onCompleteCalled.set(true); + } + }); + + try { + Thread.sleep(300); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + assertThat(onCompleteCalled).isFalse(); + assertThat(errorOccurred).isFalse(); + assertThat(events.size()).isEqualTo(1); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversionTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversionTest.java new file mode 100644 index 000000000000..902046d577d6 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversionTest.java @@ -0,0 +1,470 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.amazonaws.s3.model.GetObjectOutput; +import com.amazonaws.s3.model.PutObjectOutput; +import com.amazonaws.s3.model.ReplicationStatus; +import java.lang.reflect.Field; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.Test; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.signer.AwsS3V4Signer; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.core.ApiName; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.util.SdkUserAgent; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.metrics.LoggingMetricPublisher; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.ObjectCannedACL; +import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus; +import software.amazon.awssdk.services.s3.model.ObjectLockMode; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.RequestPayer; +import software.amazon.awssdk.services.s3.model.ServerSideEncryption; +import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.utils.Logger; + +public class S3CrtPojoConversionTest { + private static final Logger log = Logger.loggerFor(S3CrtPojoConversionTest.class); + private static final Random RNG = new Random(); + + @Test + public void fromCrtPutObjectOutputAllFields_shouldConvert() throws IllegalAccessException { + + PutObjectOutput crtResponse = randomCrtPutObjectOutput(); + SdkHttpResponse sdkHttpResponse = SdkHttpResponse.builder() + .build(); + PutObjectResponse sdkResponse = S3CrtPojoConversion.fromCrtPutObjectOutput(crtResponse, sdkHttpResponse); + + // ignoring fields with different casings and enum fields. + assertThat(sdkResponse).isEqualToIgnoringGivenFields(crtResponse, + "sseCustomerAlgorithm", + "sseCustomerKeyMD5", + "ssekmsKeyId", + "ssekmsEncryptionContext", + "serverSideEncryption", + "requestCharged", + "responseMetadata", + "sdkHttpResponse"); + assertThat(sdkResponse.serverSideEncryption().name()).isEqualTo(crtResponse.serverSideEncryption().name()); + assertThat(sdkResponse.sseCustomerAlgorithm()).isEqualTo(crtResponse.sSECustomerAlgorithm()); + assertThat(sdkResponse.ssekmsKeyId()).isEqualTo(crtResponse.sSEKMSKeyId()); + assertThat(sdkResponse.sseCustomerKeyMD5()).isEqualTo(crtResponse.sSECustomerKeyMD5()); + assertThat(sdkResponse.ssekmsEncryptionContext()).isEqualTo(crtResponse.sSEKMSEncryptionContext()); + + // TODO: CRT enums dont' have valid values. Uncomment this once it's fixed in CRT. + //assertThat(sdkResponse.requestCharged().name()).isEqualTo(crtResponse.requestCharged().name()); + } + + @Test + public void fromCrtPutObjectOutputAllFields_shouldAddSdkHttpResponse() throws IllegalAccessException { + String expectedRequestId = "123456"; + PutObjectOutput crtResponse = PutObjectOutput.builder().build(); + SdkHttpResponse sdkHttpResponse = SdkHttpResponse.builder() + .statusCode(200) + .appendHeader("x-amz-request-id", expectedRequestId) + .build(); + PutObjectResponse sdkResponse = S3CrtPojoConversion.fromCrtPutObjectOutput(crtResponse, sdkHttpResponse); + + // ignoring fields with different casing and enum fields. + assertThat(sdkResponse).isEqualToIgnoringGivenFields(crtResponse, + "sseCustomerAlgorithm", + "sseCustomerKeyMD5", + "ssekmsKeyId", + "ssekmsEncryptionContext", + "serverSideEncryption", + "requestCharged", + "responseMetadata", + "sdkHttpResponse"); + assertThat(sdkResponse.sdkHttpResponse()).isEqualTo(sdkHttpResponse); + assertThat(sdkResponse.responseMetadata().requestId()).isEqualTo(expectedRequestId); + } + + @Test + public void fromCrtGetObjectOutput_shouldAddSdkHttpResponse() { + String expectedRequestId = "123456"; + GetObjectOutput output = GetObjectOutput.builder().build(); + SdkHttpResponse response = SdkHttpResponse.builder() + .statusCode(200) + .appendHeader("x-amz-request-id", expectedRequestId) + .build(); + + + GetObjectResponse getObjectResponse = S3CrtPojoConversion.fromCrtGetObjectOutput(output, response); + assertThat(output).isEqualToIgnoringGivenFields(getObjectResponse, "body", + "sSECustomerAlgorithm", + "sSECustomerKeyMD5", + "sSEKMSKeyId", + "metadata"); + + assertThat(getObjectResponse.sdkHttpResponse()).isEqualTo(response); + assertThat(getObjectResponse.responseMetadata().requestId()).isEqualTo(expectedRequestId); + + } + + @Test + public void fromCrtGetObjectOutputAllFields_shouldConvert() throws IllegalAccessException { + GetObjectOutput crtResponse = randomCrtGetObjectOutput(); + GetObjectResponse sdkResponse = S3CrtPojoConversion.fromCrtGetObjectOutput(crtResponse, SdkHttpResponse.builder().build()); + + // ignoring fields with different casings and enum fields. + assertThat(sdkResponse).isEqualToIgnoringGivenFields(crtResponse, + "sseCustomerAlgorithm", + "body", + "sseCustomerKeyMD5", + "ssekmsKeyId", + "ssekmsEncryptionContext", + "serverSideEncryption", + "responseMetadata", + "sdkHttpResponse", + "storageClass", + "requestCharged", + "replicationStatus", + "objectLockMode", + "objectLockLegalHoldStatus"); + assertThat(sdkResponse.serverSideEncryption().name()).isEqualTo(crtResponse.serverSideEncryption().name()); + assertThat(sdkResponse.sseCustomerAlgorithm()).isEqualTo(crtResponse.sSECustomerAlgorithm()); + assertThat(sdkResponse.ssekmsKeyId()).isEqualTo(crtResponse.sSEKMSKeyId()); + assertThat(sdkResponse.sseCustomerKeyMD5()).isEqualTo(crtResponse.sSECustomerKeyMD5()); + assertThat(sdkResponse.storageClass().name()).isEqualTo(crtResponse.storageClass().name()); + assertThat(sdkResponse.replicationStatus().name()).isEqualTo(crtResponse.replicationStatus().name()); + assertThat(sdkResponse.objectLockMode().name()).isEqualTo(crtResponse.objectLockMode().name()); + assertThat(sdkResponse.objectLockLegalHoldStatus().name()).isEqualTo(crtResponse.objectLockLegalHoldStatus().name()); + + // TODO: CRT enums dont' have valid values. Uncomment this once it's fixed in CRT. + // assertThat(sdkResponse.requestCharged().name()).isEqualTo(crtResponse.requestCharged().name()); + } + + @Test + public void toCrtPutObjectRequest_shouldAddUserAgent() { + + PutObjectRequest sdkRequest = PutObjectRequest.builder() + .build(); + + com.amazonaws.s3.model.PutObjectRequest crtRequest = S3CrtPojoConversion.toCrtPutObjectRequest(sdkRequest); + HttpHeader[] headers = crtRequest.customHeaders(); + verifyUserAgent(headers); + } + + @Test + public void toCrtPutObjectRequestAllFields_shouldConvert() { + PutObjectRequest sdkRequest = randomPutObjectRequest(); + + com.amazonaws.s3.model.PutObjectRequest crtRequest = S3CrtPojoConversion.toCrtPutObjectRequest(sdkRequest); + + // ignoring fields with different casings and enum fields. + assertThat(crtRequest).isEqualToIgnoringGivenFields(sdkRequest, + "aCL", "body", "sSECustomerAlgorithm", + "sSECustomerKey", "sSECustomerKeyMD5", + "sSEKMSKeyId", "sSEKMSEncryptionContext", + "customHeaders", "customQueryParameters", + "serverSideEncryption", + "storageClass", + "requestPayer", + "objectLockMode", + "objectLockLegalHoldStatus"); + assertThat(crtRequest.aCL().name()).isEqualTo(sdkRequest.acl().name()); + assertThat(crtRequest.serverSideEncryption().name()).isEqualTo(sdkRequest.serverSideEncryption().name()); + assertThat(crtRequest.storageClass().name()).isEqualTo(sdkRequest.storageClass().name()); + assertThat(crtRequest.requestPayer().name()).isEqualTo(sdkRequest.requestPayer().name()); + assertThat(crtRequest.objectLockMode().name()).isEqualTo(sdkRequest.objectLockMode().name()); + assertThat(crtRequest.objectLockLegalHoldStatus().name()).isEqualTo(sdkRequest.objectLockLegalHoldStatus().name()); + + assertThat(crtRequest.sSECustomerAlgorithm()).isEqualTo(sdkRequest.sseCustomerAlgorithm()); + assertThat(crtRequest.sSECustomerKey()).isEqualTo(sdkRequest.sseCustomerKey()); + assertThat(crtRequest.sSECustomerKeyMD5()).isEqualTo(sdkRequest.sseCustomerKeyMD5()); + assertThat(crtRequest.sSEKMSKeyId()).isEqualTo(sdkRequest.ssekmsKeyId()); + assertThat(crtRequest.sSEKMSEncryptionContext()).isEqualTo(sdkRequest.ssekmsEncryptionContext()); + assertThat(crtRequest.sSECustomerAlgorithm()).isEqualTo(sdkRequest.sseCustomerAlgorithm()); + } + + @Test + public void toCrtPutObjectRequest_withCustomHeaders_shouldAttach() { + + AwsRequestOverrideConfiguration requestOverrideConfiguration = requestOverrideConfigWithCustomHeaders(); + + PutObjectRequest sdkRequest = PutObjectRequest.builder() + .overrideConfiguration(requestOverrideConfiguration) + .build(); + + com.amazonaws.s3.model.PutObjectRequest crtRequest = S3CrtPojoConversion.toCrtPutObjectRequest(sdkRequest); + HttpHeader[] headers = crtRequest.customHeaders(); + verifyHeaders(headers); + assertThat(crtRequest.customQueryParameters()).isEqualTo("?hello1=world1&hello2=world2"); + } + + @Test + public void toCrtGetObjectRequest_shouldAddUserAgent() { + GetObjectRequest sdkRequest = GetObjectRequest.builder() + .build(); + + com.amazonaws.s3.model.GetObjectRequest crtRequest = S3CrtPojoConversion.toCrtGetObjectRequest(sdkRequest); + + HttpHeader[] headers = crtRequest.customHeaders(); + verifyUserAgent(headers); + } + + @Test + public void toCrtGetObjectRequestAllFields_shouldConvert() { + GetObjectRequest sdkRequest = randomGetObjectRequest(); + + com.amazonaws.s3.model.GetObjectRequest crtRequest = S3CrtPojoConversion.toCrtGetObjectRequest(sdkRequest); + + // ignoring fields with different casings and enum fields. + assertThat(crtRequest).isEqualToIgnoringGivenFields(sdkRequest, "body", "sSECustomerAlgorithm", + "sSECustomerKey", "sSECustomerKeyMD5", + "customHeaders", "customQueryParameters", + "requestPayer"); + assertThat(crtRequest.requestPayer().name()).isEqualTo(sdkRequest.requestPayer().name()); + assertThat(crtRequest.sSECustomerAlgorithm()).isEqualTo(sdkRequest.sseCustomerAlgorithm()); + assertThat(crtRequest.sSECustomerKey()).isEqualTo(sdkRequest.sseCustomerKey()); + assertThat(crtRequest.sSECustomerKeyMD5()).isEqualTo(sdkRequest.sseCustomerKeyMD5()); + assertThat(crtRequest.sSECustomerAlgorithm()).isEqualTo(sdkRequest.sseCustomerAlgorithm()); + } + + @Test + public void toCrtGetObjectRequest_withCustomHeaders_shouldAttach() { + AwsRequestOverrideConfiguration requestOverrideConfiguration = requestOverrideConfigWithCustomHeaders(); + + GetObjectRequest sdkRequest = GetObjectRequest.builder() + .overrideConfiguration(requestOverrideConfiguration) + .build(); + + com.amazonaws.s3.model.GetObjectRequest crtRequest = S3CrtPojoConversion.toCrtGetObjectRequest(sdkRequest); + + HttpHeader[] headers = crtRequest.customHeaders(); + verifyHeaders(headers); + assertThat(crtRequest.customQueryParameters()).isEqualTo("?hello1=world1&hello2=world2"); + } + + @Test + public void toCrtPutObjectRequest_withUnsupportedConfigs_shouldThrowException() { + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() + .overrideConfiguration(b -> b.apiCallAttemptTimeout(Duration.ofMinutes(1))) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() + .overrideConfiguration(b -> b.apiCallTimeout(Duration.ofMinutes(1))) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() + .overrideConfiguration(b -> b.credentialsProvider(() -> + AwsBasicCredentials.create("", ""))) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() + .overrideConfiguration(b -> b.addApiName(ApiName.builder() + .name("test") + .version("1") + .build())) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() + .overrideConfiguration(b -> b.addMetricPublisher(LoggingMetricPublisher.create())) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() + .overrideConfiguration(b -> b.signer(AwsS3V4Signer.create())) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void toCrtGetObjectRequest_withUnsupportedConfigs_shouldThrowException() { + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() + .overrideConfiguration(b -> b.apiCallAttemptTimeout(Duration.ofMinutes(1))) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() + .overrideConfiguration(b -> b.apiCallTimeout(Duration.ofMinutes(1))) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() + .overrideConfiguration(b -> b.credentialsProvider(() -> + AwsBasicCredentials.create("", ""))) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() + .overrideConfiguration(b -> b.addApiName(ApiName.builder() + .name("test") + .version("1") + .build())) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() + .overrideConfiguration(b -> b.addMetricPublisher(LoggingMetricPublisher.create())) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() + .overrideConfiguration(b -> b.signer(AwsS3V4Signer.create())) + .build())).isExactlyInstanceOf(UnsupportedOperationException.class); + } + + private AwsRequestOverrideConfiguration requestOverrideConfigWithCustomHeaders() { + return AwsRequestOverrideConfiguration.builder() + .putHeader("foo", "bar") + .putRawQueryParameter("hello1", "world1") + .putRawQueryParameter("hello2", "world2") + .build(); + } + + private void verifyHeaders(HttpHeader[] headers) { + assertThat(headers).hasSize(2); + verifyUserAgent(headers); + assertThat(headers[1].getName()).isEqualTo("foo"); + assertThat(headers[1].getValue()).isEqualTo("bar"); + } + + private void verifyUserAgent(HttpHeader[] headers) { + assertThat(headers[0].getName()).isEqualTo("User-Agent"); + assertThat(headers[0].getValue()).contains("ft/s3-transfer"); + assertThat(headers[0].getValue()).contains(SdkUserAgent.create().userAgent()); + } + + private GetObjectRequest randomGetObjectRequest() { + GetObjectRequest.Builder builder = GetObjectRequest.builder(); + setSdkFieldsToRandomValues(builder.sdkFields(), builder); + return builder.build(); + } + + private PutObjectRequest randomPutObjectRequest() { + PutObjectRequest.Builder builder = PutObjectRequest.builder(); + setSdkFieldsToRandomValues(builder.sdkFields(), builder); + return builder.build(); + } + + + private com.amazonaws.s3.model.GetObjectOutput randomCrtGetObjectOutput() throws IllegalAccessException { + com.amazonaws.s3.model.GetObjectOutput.Builder builder = com.amazonaws.s3.model.GetObjectOutput.builder(); + Class aClass = builder.getClass(); + setFieldsToRandomValues(Arrays.asList(aClass.getDeclaredFields()), builder); + return builder.build(); + } + + private com.amazonaws.s3.model.PutObjectOutput randomCrtPutObjectOutput() throws IllegalAccessException { + com.amazonaws.s3.model.PutObjectOutput.Builder builder = com.amazonaws.s3.model.PutObjectOutput.builder(); + Class aClass = builder.getClass(); + setFieldsToRandomValues(Arrays.asList(aClass.getDeclaredFields()), builder); + return builder.build(); + } + + private void setFieldsToRandomValues(Collection fields, Object builder) throws IllegalAccessException { + for (Field f : fields) { + setFieldToRandomValue(f, builder); + } + } + + private void setFieldToRandomValue(Field field, Object obj) throws IllegalAccessException { + Class targetClass = field.getType(); + field.setAccessible(true); + if (targetClass.equals(String.class)) { + field.set(obj, RandomStringUtils.randomAscii(8)); + } else if (targetClass.equals(Integer.class)) { + field.set(obj, randomInteger()); + } else if (targetClass.equals(Instant.class)) { + field.set(obj, randomInstant()); + } else if (targetClass.equals(Long.class)) { + field.set(obj, RNG.nextLong()); + } else if (targetClass.equals(Map.class)) { + field.set(obj, new HashMap<>()); + } else if (targetClass.equals(Boolean.class)) { + field.set(obj, Boolean.TRUE); + } else if (targetClass.isEnum()) { + if (targetClass.equals(com.amazonaws.s3.model.ServerSideEncryption.class)) { + field.set(obj, com.amazonaws.s3.model.ServerSideEncryption.AES256); + } else if (targetClass.equals(com.amazonaws.s3.model.StorageClass.class)) { + field.set(obj, com.amazonaws.s3.model.StorageClass.GLACIER); + } else if (targetClass.equals(com.amazonaws.s3.model.RequestCharged.class)) { + field.set(obj, com.amazonaws.s3.model.RequestCharged.REQUESTER); + } else if (targetClass.equals(com.amazonaws.s3.model.ReplicationStatus.class)) { + field.set(obj, ReplicationStatus.COMPLETE); + } else if (targetClass.equals(com.amazonaws.s3.model.ObjectLockMode.class)) { + field.set(obj, com.amazonaws.s3.model.ObjectLockMode.GOVERNANCE); + } else if (targetClass.equals(com.amazonaws.s3.model.ObjectLockLegalHoldStatus.class)) { + field.set(obj, com.amazonaws.s3.model.ObjectLockLegalHoldStatus.OFF); + } else { + throw new IllegalArgumentException("Unknown enum: " + field.getName()); + } + } else if (field.getName().equals("body")) { + log.info(() -> "ignore non s3 fields"); + } else if (field.isSynthetic()) { + // ignore jacoco https://github.com/jacoco/jacoco/issues/168 + log.info(() -> "ignore synthetic fields"); + } else { + throw new IllegalArgumentException("Unknown Field type: " + field.getName()); + } + } + + private void setSdkFieldsToRandomValues(Collection> fields, Object builder) { + for (SdkField f : fields) { + setSdkFieldToRandomValue(f, builder); + } + } + + private static void setSdkFieldToRandomValue(SdkField sdkField, Object obj) { + Class targetClass = sdkField.marshallingType().getTargetClass(); + if (targetClass.equals(String.class)) { + switch (sdkField.memberName()) { + case "ACL": + sdkField.set(obj, ObjectCannedACL.PUBLIC_READ.toString()); + break; + case "ServerSideEncryption": + sdkField.set(obj, ServerSideEncryption.AES256.toString()); + break; + case "StorageClass": + sdkField.set(obj, StorageClass.DEEP_ARCHIVE.toString()); + break; + case "RequestPayer": + sdkField.set(obj, RequestPayer.UNKNOWN_TO_SDK_VERSION.toString()); + break; + case "ObjectLockMode": + sdkField.set(obj, ObjectLockMode.COMPLIANCE.toString()); + break; + case "ObjectLockLegalHoldStatus": + sdkField.set(obj, ObjectLockLegalHoldStatus.OFF.toString()); + break; + default: + sdkField.set(obj, RandomStringUtils.random(8)); + } + } else if (targetClass.equals(Integer.class)) { + sdkField.set(obj, randomInteger()); + } else if (targetClass.equals(Instant.class)) { + sdkField.set(obj, randomInstant()); + } else if (targetClass.equals(Long.class)) { + sdkField.set(obj, RNG.nextLong()); + } else if (targetClass.equals(Map.class)) { + sdkField.set(obj, new HashMap<>()); + } else if (targetClass.equals(Boolean.class)) { + sdkField.set(obj, Boolean.TRUE); + } else { + throw new IllegalArgumentException("Unknown SdkField type: " + targetClass); + } + } + + private static Instant randomInstant() { + return Instant.ofEpochMilli(RNG.nextLong()); + } + + private static Integer randomInteger() { + return RNG.nextInt(); + } +} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfigurationTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfigurationTest.java new file mode 100644 index 000000000000..13c4584a2564 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3NativeClientConfigurationTest.java @@ -0,0 +1,68 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR; + +import java.util.concurrent.ExecutorService; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; +import software.amazon.awssdk.crt.CrtResource; + +@RunWith(MockitoJUnitRunner.class) +public class S3NativeClientConfigurationTest { + + @Mock + private ExecutorService executorService; + + @BeforeClass + public static void setup() { + System.setProperty("aws.crt.debugnative", "true"); + } + + @AfterClass + public static void tearDown() { + CrtResource.waitForNoResources(); + } + + @Test + public void defaultConfiguration_close_shouldShutdownDefaultExecutor() { + S3NativeClientConfiguration configuration = S3NativeClientConfiguration.builder().build(); + assertThat(((ExecutorService) configuration.futureCompletionExecutor()).isShutdown()).isFalse(); + configuration.close(); + assertThat(configuration.futureCompletionExecutor()).isInstanceOf(ExecutorService.class); + assertThat(((ExecutorService) configuration.futureCompletionExecutor()).isShutdown()).isTrue(); + } + + @Test + public void customExecutor_close_shouldNotShutdownCustomExecutor() { + S3NativeClientConfiguration configuration = S3NativeClientConfiguration.builder() + .asyncConfiguration(ClientAsyncConfiguration.builder() + .advancedOption(FUTURE_COMPLETION_EXECUTOR, executorService) + .build()) + .build(); + + configuration.close(); + Mockito.verifyZeroInteractions(executorService); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java new file mode 100644 index 000000000000..9433edd10a79 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java @@ -0,0 +1,126 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import java.nio.file.Paths; +import java.util.concurrent.CompletableFuture; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.transfer.s3.CompletedDownload; +import software.amazon.awssdk.transfer.s3.CompletedUpload; +import software.amazon.awssdk.transfer.s3.DownloadRequest; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.UploadRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +public class S3TransferManagerTest { + private S3CrtAsyncClient mockS3Crt; + private S3TransferManager tm; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Before + public void methodSetup() { + mockS3Crt = mock(S3CrtAsyncClient.class); + tm = new DefaultS3TransferManager(mockS3Crt); + } + + @After + public void methodTeardown() { + tm.close(); + } + + @Test + public void upload_returnsResponse() { + PutObjectResponse response = PutObjectResponse.builder().build(); + when(mockS3Crt.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + CompletedUpload completedUpload = tm.upload(UploadRequest.builder() + .putObjectRequest(r -> r.bucket("bucket") + .key("key")) + .source(Paths.get(".")) + .build()) + .completionFuture() + .join(); + + assertThat(completedUpload.response()).isEqualTo(response); + } + + @Test + public void upload_cancel_shouldForwardCancellation() { + CompletableFuture s3CrtFuture = new CompletableFuture<>(); + when(mockS3Crt.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class))) + .thenReturn(s3CrtFuture); + + CompletableFuture future = tm.upload(UploadRequest.builder() + .putObjectRequest(r -> r.bucket("bucket") + .key("key")) + .source(Paths.get(".")) + .build()) + .completionFuture(); + + future.cancel(true); + assertThat(s3CrtFuture).isCancelled(); + } + + @Test + public void download_returnsResponse() { + GetObjectResponse response = GetObjectResponse.builder().build(); + when(mockS3Crt.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + CompletedDownload completedDownload = tm.download(DownloadRequest.builder() + .getObjectRequest(r -> r.bucket("bucket") + .key("key")) + .destination(Paths.get(".")) + .build()) + .completionFuture() + .join(); + assertThat(completedDownload.response()).isEqualTo(response); + } + + @Test + public void download_cancel_shouldForwardCancellation() { + CompletableFuture s3CrtFuture = new CompletableFuture<>(); + when(mockS3Crt.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .thenReturn(s3CrtFuture); + + CompletableFuture future = tm.download(DownloadRequest.builder() + .getObjectRequest(r -> r.bucket("bucket") + .key("key")) + .destination(Paths.get(".")) + .build()) + .completionFuture(); + future.cancel(true); + assertThat(s3CrtFuture).isCancelled(); + } + + +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/util/ChecksumUtils.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/util/ChecksumUtils.java new file mode 100644 index 000000000000..df0f93ed9420 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/util/ChecksumUtils.java @@ -0,0 +1,69 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.util; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +/** + * Utilities for computing the SHA-256 checksums of various binary objects. + */ +public final class ChecksumUtils { + public static byte[] computeCheckSum(InputStream is) throws IOException { + MessageDigest instance = createMessageDigest(); + + byte buff[] = new byte[16384]; + int read; + while ((read = is.read(buff)) != -1) { + instance.update(buff, 0, read); + } + + return instance.digest(); + } + + public static byte[] computeCheckSum(ByteBuffer bb) { + MessageDigest instance = createMessageDigest(); + + instance.update(bb); + + bb.rewind(); + + return instance.digest(); + } + + public static byte[] computeCheckSum(List buffers) { + MessageDigest instance = createMessageDigest(); + + buffers.forEach(bb -> { + instance.update(bb); + bb.rewind(); + }); + + return instance.digest(); + } + + private static MessageDigest createMessageDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Unable to create SHA-256 MessageDigest instance", e); + } + } +} diff --git a/services-custom/s3-transfer-manager/src/test/resources/log4j.properties b/services-custom/s3-transfer-manager/src/test/resources/log4j.properties new file mode 100644 index 000000000000..b821297c6731 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/resources/log4j.properties @@ -0,0 +1,33 @@ +# +# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.appender.A1=org.apache.log4j.ConsoleAppender +log4j.appender.A1.layout=org.apache.log4j.PatternLayout + +# Print the date in ISO 8601 format +log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n + +# Adjust to see more / less logging +#log4j.logger.com.amazonaws.ec2=DEBUG + +# HttpClient 3 Wire Logging +#log4j.logger.httpclient.wire=DEBUG + +# HttpClient 4 Wire Logging +#log4j.logger.org.apache.http.wire=DEBUG +#log4j.logger.org.apache.http=DEBUG +#log4j.logger.org.apache.http.wire=WARN +#log4j.logger.software.amazon.awssdk=DEBUG diff --git a/services/s3/pom.xml b/services/s3/pom.xml index 43cb6912bd18..06f03942083b 100644 --- a/services/s3/pom.xml +++ b/services/s3/pom.xml @@ -106,5 +106,10 @@ wiremock test + + org.reactivestreams + reactive-streams-tck + test + diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/utils/ChecksumUtils.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/utils/ChecksumUtils.java new file mode 100644 index 000000000000..28dcf23340d8 --- /dev/null +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/utils/ChecksumUtils.java @@ -0,0 +1,69 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +/** + * Utilities for computing the SHA-256 checksums of various binary objects. + */ +public final class ChecksumUtils { + public static byte[] computeCheckSum(InputStream is) throws IOException { + MessageDigest instance = createMessageDigest(); + + byte buff[] = new byte[16384]; + int read; + while ((read = is.read(buff)) != -1) { + instance.update(buff, 0, read); + } + + return instance.digest(); + } + + public static byte[] computeCheckSum(ByteBuffer bb) { + MessageDigest instance = createMessageDigest(); + + instance.update(bb); + + bb.rewind(); + + return instance.digest(); + } + + public static byte[] computeCheckSum(List buffers) { + MessageDigest instance = createMessageDigest(); + + buffers.forEach(bb -> { + instance.update(bb); + bb.rewind(); + }); + + return instance.digest(); + } + + private static MessageDigest createMessageDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Unable to create SHA-256 MessageDigest instance", e); + } + } +} diff --git a/test/auth-sts-testing/src/it/java/software/amazon/awssdk/authststesting/ProfileCredentialsProviderIntegrationTest.java b/test/auth-sts-testing/src/it/java/software/amazon/awssdk/authststesting/ProfileCredentialsProviderIntegrationTest.java index 104694cb7414..7d56f164fa37 100644 --- a/test/auth-sts-testing/src/it/java/software/amazon/awssdk/authststesting/ProfileCredentialsProviderIntegrationTest.java +++ b/test/auth-sts-testing/src/it/java/software/amazon/awssdk/authststesting/ProfileCredentialsProviderIntegrationTest.java @@ -30,7 +30,7 @@ import java.time.Instant; import org.junit.Test; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.core.internal.util.UserAgentUtils; +import software.amazon.awssdk.core.util.SdkUserAgent; import software.amazon.awssdk.profiles.ProfileFile; import software.amazon.awssdk.services.sts.model.StsException; import software.amazon.awssdk.utils.DateUtils; @@ -80,7 +80,7 @@ public void profileWithCredentialSourceUsingEc2InstanceMetadataAndCustomEndpoint } String userAgentHeader = "User-Agent"; - String userAgent = UserAgentUtils.getUserAgent(); + String userAgent = SdkUserAgent.create().userAgent(); mockMetadataEndpoint.verify(putRequestedFor(urlPathEqualTo(TOKEN_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent))); mockMetadataEndpoint.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent))); mockMetadataEndpoint.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH + "some-profile")).withHeader(userAgentHeader, equalTo(userAgent))); diff --git a/test/s3-benchmarks/README.md b/test/s3-benchmarks/README.md new file mode 100755 index 000000000000..21d86f7d75d6 --- /dev/null +++ b/test/s3-benchmarks/README.md @@ -0,0 +1,18 @@ +# S3 Benchmark Harness + + +This module contains performance tests for `S3AsyncClient` and +`S3TransferManager` + +## How to run + +``` +# Build the JAR +mvn clean install -pl :s3-benchmarks -P quick --am + +# download +java -jar s3-benchmarks.jar --bucket=bucket --key=key -file=/path/to/destionfile/ --operation=download --partSizeInMB=20 --maxThroughput=100.0 + +# upload +java -jar s3-benchmarks.jar --bucket=bucket --key=key -file=/path/to/sourcefile/ --operation=upload --partSizeInMB=20 --maxThroughput=100.0 +``` diff --git a/test/s3-benchmarks/pom.xml b/test/s3-benchmarks/pom.xml new file mode 100644 index 000000000000..4e099250fe6d --- /dev/null +++ b/test/s3-benchmarks/pom.xml @@ -0,0 +1,128 @@ + + + + + + aws-sdk-java-pom + software.amazon.awssdk + 2.17.16-SNAPSHOT + ../../pom.xml + + 4.0.0 + + s3-benchmarks + + 8 + 8 + ${project.version} + + AWS Java SDK :: Test :: S3 Benchmarks + Contains benchmark code for S3 and TransferManager + + + + software.amazon.awssdk + bom-internal + ${project.version} + pom + import + + + + + + + commons-cli + commons-cli + + + software.amazon.awssdk + s3-transfer-manager + ${awsjavasdk.version}-PREVIEW + + + software.amazon.awssdk + test-utils + ${awsjavasdk.version} + compile + + + log4j + log4j + compile + + + org.slf4j + slf4j-log4j12 + compile + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + build + package + + single + + + + + jar-with-dependencies + + + + false + + ${project.artifactId} + + + true + lib/ + software.amazon.awssdk.s3benchmarks.BenchmarkRunner + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + analyze-only + + + + + + true + + + + + + \ No newline at end of file diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BaseTransferManagerBenchmark.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BaseTransferManagerBenchmark.java new file mode 100644 index 000000000000..6b505c9b4989 --- /dev/null +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BaseTransferManagerBenchmark.java @@ -0,0 +1,158 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.s3benchmarks; + +import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient; +import software.amazon.awssdk.utils.Logger; + +public abstract class BaseTransferManagerBenchmark implements TransferManagerBenchmark { + protected static final int WARMUP_ITERATIONS = 10; + protected static final int BENCHMARK_ITERATIONS = 10; + + private static final Logger logger = Logger.loggerFor("TransferManagerBenchmark"); + private static final String WARMUP_KEY = "warmupobject"; + + protected final S3TransferManager transferManager; + protected final S3CrtAsyncClient s3; + protected final S3Client s3Sync; + protected final String bucket; + protected final String key; + protected final String path; + private final File file; + + BaseTransferManagerBenchmark(TransferManagerBenchmarkConfig config) { + logger.info(() -> "Benchmark config: " + config); + Long partSizeInMb = config.partSizeInMb() == null ? null : config.partSizeInMb() * 1024 * 1024L; + s3 = S3CrtAsyncClient.builder() + .targetThroughputInGbps(config.targetThroughput()) + .minimumPartSizeInBytes(partSizeInMb) + .build(); + s3Sync = S3Client.builder() + .build(); + transferManager = S3TransferManager.builder() + .s3ClientConfiguration(b -> b.targetThroughputInGbps(config.targetThroughput()) + .minimumPartSizeInBytes(partSizeInMb)) + .build(); + bucket = config.bucket(); + key = config.key(); + path = config.filePath(); + try { + file = new RandomTempFile(1024 * 1000L); + } catch (IOException e) { + logger.error(() -> "Failed to create the file"); + throw new RuntimeException("Failed to create the temp file", e); + } + } + + @Override + public void run() { + try { + warmUp(); + doRunBenchmark(); + } catch (Exception e) { + logger.error(() -> "Exception occurred", e); + } finally { + cleanup(); + } + } + + /** + * Hook method to allow subclasses to add additional warm up + */ + protected void additionalWarmup() { + // default to no-op + } + + protected abstract void doRunBenchmark(); + + protected final void printOutResult(List metrics, String name) { + logger.info(() -> String.format("=============== %s Result ================", name)); + logger.info(() -> "" + metrics); + double averageLatency = metrics.stream() + .mapToDouble(a -> a) + .average() + .orElse(0.0); + + double lowestLatency = metrics.stream() + .mapToDouble(a -> a) + .min().orElse(0.0); + + HeadObjectResponse headObjectResponse = s3Sync.headObject(b -> b.bucket(bucket).key(key)); + double contentLengthInGigabit = (headObjectResponse.contentLength() / (1000 * 1000 * 1000.0)) * 8.0; + logger.info(() -> "Average latency (s): " + averageLatency); + logger.info(() -> "Object size (Gigabit): " + contentLengthInGigabit); + logger.info(() -> "Average throughput (Gbps): " + contentLengthInGigabit / averageLatency); + logger.info(() -> "Highest average throughput (Gbps): " + contentLengthInGigabit / lowestLatency); + logger.info(() -> "=========================================================="); + } + + private void cleanup() { + s3Sync.deleteObject(b -> b.bucket(bucket).key(WARMUP_KEY)); + transferManager.close(); + } + + private void warmUp() throws InterruptedException { + logger.info(() -> "Starting to warm up"); + + for (int i = 0; i < WARMUP_ITERATIONS; i++) { + warmUpUploadBatch(); + warmUpDownloadBatch(); + + Thread.sleep(500); + } + additionalWarmup(); + logger.info(() -> "Ending warm up"); + } + + private void warmUpDownloadBatch() { + List> futures = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + Path tempFile = RandomTempFile.randomUncreatedFile().toPath(); + futures.add(s3.getObject(GetObjectRequest.builder().bucket(bucket).key(WARMUP_KEY).build(), + AsyncResponseTransformer.toFile(tempFile)).whenComplete((r, t) -> runAndLogError( + logger.logger(), "Deleting file failed", () -> Files.delete(tempFile)))); + } + + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + } + + private void warmUpUploadBatch() { + List> futures = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + futures.add(s3.putObject(PutObjectRequest.builder().bucket(bucket).key(WARMUP_KEY).build(), + AsyncRequestBody.fromFile(file))); + } + + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + } +} diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BenchmarkRunner.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BenchmarkRunner.java new file mode 100644 index 000000000000..209396ed89eb --- /dev/null +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/BenchmarkRunner.java @@ -0,0 +1,88 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.s3benchmarks; + +import java.util.Locale; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; + +public class BenchmarkRunner { + + private static final String PART_SIZE_IN_MB = "partSizeInMB"; + private static final String FILE = "file"; + private static final String BUCKET = "bucket"; + private static final String MAX_THROUGHPUT = "maxThroughput"; + private static final String KEY = "key"; + private static final String OPERATION = "operation"; + + private BenchmarkRunner() { + } + + public static void main(String... args) throws org.apache.commons.cli.ParseException { + CommandLineParser parser = new DefaultParser(); + + Options options = new Options(); + + options.addRequiredOption(null, FILE, true, "Destination file path to be written or source file path to be " + + "uploaded"); + options.addRequiredOption(null, BUCKET, true, "The s3 bucket"); + options.addRequiredOption(null, KEY, true, "The s3 key"); + options.addRequiredOption(null, OPERATION, true, "The operation to benchmark against"); + options.addOption(null, PART_SIZE_IN_MB, true, "Part size in MB"); + options.addOption(null, MAX_THROUGHPUT, true, "The max throughput"); + + CommandLine cmd = parser.parse(options, args); + TransferManagerBenchmarkConfig config = parseConfig(cmd); + TransferManagerOperation operation = TransferManagerOperation.valueOf(cmd.getOptionValue(OPERATION) + .toUpperCase(Locale.ENGLISH)); + switch (operation) { + case DOWNLOAD: + TransferManagerBenchmark.download(config).run(); + break; + case UPLOAD: + TransferManagerBenchmark.upload(config).run(); + break; + default: + throw new UnsupportedOperationException(); + } + } + + private static TransferManagerBenchmarkConfig parseConfig(CommandLine cmd) { + String filePath = cmd.getOptionValue(FILE); + String bucket = cmd.getOptionValue(BUCKET); + String key = cmd.getOptionValue(KEY); + + Long partSize = cmd.getOptionValue(PART_SIZE_IN_MB) == null ? null : Long.parseLong(cmd.getOptionValue(PART_SIZE_IN_MB)); + + Double maxThroughput = cmd.getOptionValue(MAX_THROUGHPUT) == null ? null : + Double.parseDouble(cmd.getOptionValue(MAX_THROUGHPUT)); + + return TransferManagerBenchmarkConfig.builder() + .key(key) + .bucket(bucket) + .partSizeInMb(partSize) + .targetThroughput(maxThroughput) + .filePath(filePath) + .build(); + } + + private enum TransferManagerOperation { + DOWNLOAD, + UPLOAD + } +} diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/NoOpResponseTransformer.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/NoOpResponseTransformer.java new file mode 100644 index 000000000000..73ba0b112ae7 --- /dev/null +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/NoOpResponseTransformer.java @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.s3benchmarks; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.http.async.SimpleSubscriber; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +/** + * A no-op {@link AsyncResponseTransformer} + */ +public class NoOpResponseTransformer implements AsyncResponseTransformer { + private CompletableFuture future; + + @Override + public CompletableFuture prepare() { + future = new CompletableFuture<>(); + return future; + } + + @Override + public void onResponse(GetObjectResponse response) { + } + + @Override + public void onStream(SdkPublisher publisher) { + publisher.subscribe(new SimpleSubscriber(Buffer::clear) { + @Override + public void onComplete() { + super.onComplete(); + future.complete(null); + } + }); + } + + @Override + public void exceptionOccurred(Throwable error) { + future.completeExceptionally(error); + } +} diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerBenchmark.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerBenchmark.java new file mode 100644 index 000000000000..e5a28a820fd3 --- /dev/null +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerBenchmark.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.s3benchmarks; + +/** + * Factory to create the benchmark + */ +@FunctionalInterface +public interface TransferManagerBenchmark { + + /** + * The benchmark method to run + */ + void run(); + + static TransferManagerBenchmark download(TransferManagerBenchmarkConfig config) { + return new TransferManagerDownloadBenchmark(config); + } + + static TransferManagerBenchmark upload(TransferManagerBenchmarkConfig config) { + return new TransferManagerUploadBenchmark(config); + } + +} diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerBenchmarkConfig.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerBenchmarkConfig.java new file mode 100644 index 000000000000..22191e4f7bf8 --- /dev/null +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerBenchmarkConfig.java @@ -0,0 +1,104 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.s3benchmarks; + +public class TransferManagerBenchmarkConfig { + private final String filePath; + private final String bucket; + private final String key; + private final Double targetThroughput; + private final Long partSizeInMb; + + private TransferManagerBenchmarkConfig(Builder builder) { + this.filePath = builder.filePath; + this.bucket = builder.bucket; + this.key = builder.key; + this.targetThroughput = builder.targetThroughput; + this.partSizeInMb = builder.partSizeInMb; + } + + public String filePath() { + return filePath; + } + + public String bucket() { + return bucket; + } + + public String key() { + return key; + } + + public Double targetThroughput() { + return targetThroughput; + } + + public Long partSizeInMb() { + return partSizeInMb; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public String toString() { + return "{" + + "filePath: '" + filePath + '\'' + + ", bucket: '" + bucket + '\'' + + ", key: '" + key + '\'' + + ", targetThroughput: " + targetThroughput + + ", partSizeInMB: " + partSizeInMb + + '}'; + } + + static final class Builder { + private String filePath; + private String bucket; + private String key; + private Double targetThroughput; + private Long partSizeInMb; + + public Builder filePath(String filePath) { + this.filePath = filePath; + return this; + } + + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + public Builder key(String key) { + this.key = key; + return this; + } + + public Builder targetThroughput(Double targetThroughput) { + this.targetThroughput = targetThroughput; + return this; + } + + public Builder partSizeInMb(Long partSizeInMb) { + this.partSizeInMb = partSizeInMb; + return this; + } + + public TransferManagerBenchmarkConfig build() { + return new TransferManagerBenchmarkConfig(this); + } + } +} diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerDownloadBenchmark.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerDownloadBenchmark.java new file mode 100644 index 000000000000..9126c48ee2ec --- /dev/null +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerDownloadBenchmark.java @@ -0,0 +1,96 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.s3benchmarks; + +import static software.amazon.awssdk.utils.FunctionalUtils.runAndLogError; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.transfer.s3.Download; +import software.amazon.awssdk.utils.Logger; + +public class TransferManagerDownloadBenchmark extends BaseTransferManagerBenchmark { + private static final Logger logger = Logger.loggerFor("TransferManagerDownloadBenchmark"); + + public TransferManagerDownloadBenchmark(TransferManagerBenchmarkConfig config) { + super(config); + } + + @Override + protected void doRunBenchmark() { + try { + downloadToFile(BENCHMARK_ITERATIONS, true); + downloadToMemory(BENCHMARK_ITERATIONS, true); + } catch (Exception exception) { + logger.error(() -> "Request failed: ", exception); + } + } + + @Override + protected void additionalWarmup() { + downloadToMemory(3, false); + downloadToFile(3, false); + } + + private void downloadToMemory(int count, boolean printoutResult) { + List metrics = new ArrayList<>(); + logger.info(() -> "Starting to download to memory"); + for (int i = 0; i < count; i++) { + downloadOnceToMemory(metrics); + } + + if (printoutResult) { + printOutResult(metrics, "Download to Memory"); + } + } + + private void downloadToFile(int count, boolean printoutResult) { + List metrics = new ArrayList<>(); + logger.info(() -> "Starting to download to file"); + for (int i = 0; i < count; i++) { + downloadOnceToFile(metrics); + } + if (printoutResult) { + printOutResult(metrics, "Download to File"); + } + } + + private void downloadOnceToFile(List latencies) { + Path downloadPath = new File(this.path).toPath(); + long start = System.currentTimeMillis(); + Download download = + transferManager.download(b -> b.getObjectRequest(r -> r.bucket(bucket).key(key)) + .destination(downloadPath)); + download.completionFuture().join(); + long end = System.currentTimeMillis(); + latencies.add((end - start) / 1000.0); + runAndLogError(logger.logger(), + "Deleting file failed", + () -> Files.delete(downloadPath)); + } + + private void downloadOnceToMemory(List latencies) { + long start = System.currentTimeMillis(); + s3.getObject(GetObjectRequest.builder().bucket(bucket).key(key).build(), + new NoOpResponseTransformer()).join(); + long end = System.currentTimeMillis(); + latencies.add((end - start) / 1000.0); + } +} diff --git a/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerUploadBenchmark.java b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerUploadBenchmark.java new file mode 100644 index 000000000000..39e9bf69ff4b --- /dev/null +++ b/test/s3-benchmarks/src/main/java/software/amazon/awssdk/s3benchmarks/TransferManagerUploadBenchmark.java @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.s3benchmarks; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import software.amazon.awssdk.utils.Logger; + +public class TransferManagerUploadBenchmark extends BaseTransferManagerBenchmark { + private static final Logger logger = Logger.loggerFor("TransferManagerUploadBenchmark"); + + public TransferManagerUploadBenchmark(TransferManagerBenchmarkConfig config) { + super(config); + } + + @Override + protected void doRunBenchmark() { + try { + uploadFromFile(BENCHMARK_ITERATIONS, true); + } catch (Exception exception) { + logger.error(() -> "Request failed: ", exception); + } + } + + @Override + protected void additionalWarmup() { + uploadFromFile(3, false); + } + + private void uploadFromFile(int count, boolean printOutResult) { + List metrics = new ArrayList<>(); + logger.info(() -> "Starting to upload from file"); + for (int i = 0; i < count; i++) { + uploadOnceFromFile(metrics); + } + if (printOutResult) { + printOutResult(metrics, "Upload from File"); + } + } + + private void uploadOnceFromFile(List latencies) { + File sourceFile = new File(path); + long start = System.currentTimeMillis(); + transferManager.upload(b -> b.putObjectRequest(r -> r.bucket(bucket).key(key)) + .source(sourceFile.toPath())) + .completionFuture().join(); + long end = System.currentTimeMillis(); + latencies.add((end - start) / 1000.0); + } +} diff --git a/test/s3-benchmarks/src/main/resources/log4j.properties b/test/s3-benchmarks/src/main/resources/log4j.properties new file mode 100644 index 000000000000..9ef03e1a7a28 --- /dev/null +++ b/test/s3-benchmarks/src/main/resources/log4j.properties @@ -0,0 +1,23 @@ +# +# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +log4j.rootLogger=INFO, A1 +log4j.appender.A1=org.apache.log4j.ConsoleAppender +log4j.appender.A1.layout=org.apache.log4j.PatternLayout + +# Print the date in ISO 8601 format +log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.logger.software.amazon.awssdk.testutils.Waiter=ERROR +log4j.logger.FileAsyncResponseTransformer=ERROR diff --git a/test/stability-tests/pom.xml b/test/stability-tests/pom.xml index d1b2402048e6..a954432cce26 100644 --- a/test/stability-tests/pom.xml +++ b/test/stability-tests/pom.xml @@ -117,6 +117,18 @@ ${awsjavasdk.version} test + + software.amazon.awssdk.crt + aws-crt + ${awscrt.version} + test + + + software.amazon.awssdk + s3-transfer-manager + ${awsjavasdk.version}-PREVIEW + test + software.amazon.awssdk transcribestreaming diff --git a/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3BaseStabilityTest.java b/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3BaseStabilityTest.java index 2bdcd1f13c6d..df8c1871f56a 100644 --- a/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3BaseStabilityTest.java +++ b/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3BaseStabilityTest.java @@ -15,6 +15,8 @@ package software.amazon.awssdk.stability.tests.s3; +import static org.assertj.core.api.Assertions.assertThat; + import java.io.File; import java.io.IOException; import java.nio.file.Path; @@ -23,7 +25,6 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.function.IntFunction; - import org.apache.commons.lang3.RandomStringUtils; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; @@ -34,10 +35,13 @@ import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.stability.tests.exceptions.StabilityTestsRetryableException; +import software.amazon.awssdk.stability.tests.utils.RetryableTest; import software.amazon.awssdk.stability.tests.utils.StabilityTestRunner; import software.amazon.awssdk.testutils.RandomTempFile; import software.amazon.awssdk.testutils.service.AwsTestBase; import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Md5Utils; public abstract class S3BaseStabilityTest extends AwsTestBase { private static final Logger log = Logger.loggerFor(S3BaseStabilityTest.class); @@ -46,6 +50,7 @@ public abstract class S3BaseStabilityTest extends AwsTestBase { protected static final String LARGE_KEY_NAME = "2GB"; protected static S3Client s3ApacheClient; + private final S3AsyncClient testClient; static { s3ApacheClient = S3Client.builder() @@ -56,16 +61,31 @@ public abstract class S3BaseStabilityTest extends AwsTestBase { .build(); } + public S3BaseStabilityTest(S3AsyncClient testClient) { + this.testClient = testClient; + } + + @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class) + public void largeObject_put_get_usingFile() { + String md5Upload = uploadLargeObjectFromFile(); + String md5Download = downloadLargeObjectToFile(); + assertThat(md5Upload).isEqualTo(md5Download); + } + + @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class) + public void putObject_getObject_highConcurrency() { + putObject(); + getObject(); + } + protected String computeKeyName(int i) { return "key_" + i; } - protected abstract S3AsyncClient getTestClient(); - protected abstract String getTestBucketName(); protected void doGetBucketAcl_lowTpsLongInterval() { - IntFunction> future = i -> getTestClient().getBucketAcl(b -> b.bucket(getTestBucketName())); + IntFunction> future = i -> testClient.getBucketAcl(b -> b.bucket(getTestBucketName())); String className = this.getClass().getSimpleName(); StabilityTestRunner.newRunner() .testName(className + ".getBucketAcl_lowTpsLongInterval") @@ -77,25 +97,35 @@ protected void doGetBucketAcl_lowTpsLongInterval() { } - protected void downloadLargeObjectToFile() { + protected String downloadLargeObjectToFile() { File randomTempFile = RandomTempFile.randomUncreatedFile(); StabilityTestRunner.newRunner() .testName("S3AsyncStabilityTest.downloadLargeObjectToFile") - .futures(getTestClient().getObject(b -> b.bucket(getTestBucketName()).key(LARGE_KEY_NAME), + .futures(testClient.getObject(b -> b.bucket(getTestBucketName()).key(LARGE_KEY_NAME), AsyncResponseTransformer.toFile(randomTempFile))) .run(); - randomTempFile.delete(); + + + try { + return Md5Utils.md5AsBase64(randomTempFile); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + randomTempFile.delete(); + } } - protected void uploadLargeObjectFromFile() { + protected String uploadLargeObjectFromFile() { RandomTempFile file = null; try { file = new RandomTempFile((long) 2e+9); + String md5 = Md5Utils.md5AsBase64(file); StabilityTestRunner.newRunner() .testName("S3AsyncStabilityTest.uploadLargeObjectFromFile") - .futures(getTestClient().putObject(b -> b.bucket(getTestBucketName()).key(LARGE_KEY_NAME), + .futures(testClient.putObject(b -> b.bucket(getTestBucketName()).key(LARGE_KEY_NAME), AsyncRequestBody.fromFile(file))) .run(); + return md5; } catch (IOException e) { throw new RuntimeException("fail to create test file", e); } finally { @@ -110,7 +140,7 @@ protected void putObject() { IntFunction> future = i -> { String keyName = computeKeyName(i); - return getTestClient().putObject(b -> b.bucket(getTestBucketName()).key(keyName), + return testClient.putObject(b -> b.bucket(getTestBucketName()).key(keyName), AsyncRequestBody.fromBytes(bytes)); }; @@ -127,7 +157,7 @@ protected void getObject() { IntFunction> future = i -> { String keyName = computeKeyName(i); Path path = RandomTempFile.randomUncreatedFile().toPath(); - return getTestClient().getObject(b -> b.bucket(getTestBucketName()).key(keyName), AsyncResponseTransformer.toFile(path)); + return testClient.getObject(b -> b.bucket(getTestBucketName()).key(keyName), AsyncResponseTransformer.toFile(path)); }; StabilityTestRunner.newRunner() diff --git a/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3CrtAsyncClientStabilityTest.java b/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3CrtAsyncClientStabilityTest.java new file mode 100644 index 000000000000..ce85d544dff1 --- /dev/null +++ b/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3CrtAsyncClientStabilityTest.java @@ -0,0 +1,68 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.stability.tests.s3; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.transfer.s3.internal.S3CrtAsyncClient; + +/** + * Stability tests for {@link S3CrtAsyncClient} + */ +public class S3CrtAsyncClientStabilityTest extends S3BaseStabilityTest { + private static final String BUCKET_NAME = "s3crtasyncclinetstabilitytests" + System.currentTimeMillis(); + private static S3CrtAsyncClient s3CrtAsyncClient; + + static { + s3CrtAsyncClient = S3CrtAsyncClient.builder() + .region(Region.US_WEST_2) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .build(); + } + + public S3CrtAsyncClientStabilityTest() { + super(s3CrtAsyncClient); + } + + @BeforeAll + public static void setup() { + System.setProperty("aws.crt.debugnative", "true"); + s3ApacheClient.createBucket(b -> b.bucket(BUCKET_NAME)); + } + + @AfterAll + public static void cleanup() { + try (S3AsyncClient s3NettyClient = S3AsyncClient.builder() + .httpClientBuilder(NettyNioAsyncHttpClient.builder() + .maxConcurrency(CONCURRENCY)) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .build()) { + deleteBucketAndAllContents(s3NettyClient, BUCKET_NAME); + } + s3CrtAsyncClient.close(); + s3ApacheClient.close(); + CrtResource.waitForNoResources(); + } + + @Override + protected String getTestBucketName() { + return BUCKET_NAME; + } +} diff --git a/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3NettyAsyncStabilityTest.java b/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3NettyAsyncStabilityTest.java index 54fd65ddfa39..d7afa62bade0 100644 --- a/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3NettyAsyncStabilityTest.java +++ b/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3NettyAsyncStabilityTest.java @@ -27,6 +27,10 @@ public class S3NettyAsyncStabilityTest extends S3BaseStabilityTest { .build(); } + public S3NettyAsyncStabilityTest() { + super(s3NettyClient); + } + @BeforeAll public static void setup() { s3NettyClient.createBucket(b -> b.bucket(bucketName)).join(); @@ -38,24 +42,9 @@ public static void cleanup() { s3NettyClient.close(); } - @Override - protected S3AsyncClient getTestClient() { return s3NettyClient; } - @Override protected String getTestBucketName() { return bucketName; } - @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class) - public void putObject_getObject_highConcurrency() { - putObject(); - getObject(); - } - - @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class) - public void largeObject_put_get_usingFile() { - uploadLargeObjectFromFile(); - downloadLargeObjectToFile(); - } - @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class) public void getBucketAcl_lowTpsLongInterval_Netty() { doGetBucketAcl_lowTpsLongInterval(); diff --git a/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3CrtAsyncStabilityTest.java b/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3WithCrtAsyncHttpClientStabilityTest.java similarity index 69% rename from test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3CrtAsyncStabilityTest.java rename to test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3WithCrtAsyncHttpClientStabilityTest.java index e8d9dd7dbb10..c92e5f691194 100644 --- a/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3CrtAsyncStabilityTest.java +++ b/test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3WithCrtAsyncHttpClientStabilityTest.java @@ -1,21 +1,21 @@ package software.amazon.awssdk.stability.tests.s3; +import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import software.amazon.awssdk.core.retry.RetryPolicy; -import software.amazon.awssdk.crt.io.EventLoopGroup; -import software.amazon.awssdk.crt.io.HostResolver; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.http.crt.AwsCrtAsyncHttpClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.stability.tests.exceptions.StabilityTestsRetryableException; import software.amazon.awssdk.stability.tests.utils.RetryableTest; -import java.time.Duration; - -public class S3CrtAsyncStabilityTest extends S3BaseStabilityTest { +/** + * Stability tests for {@link S3AsyncClient} using {@link AwsCrtAsyncHttpClient} + */ +public class S3WithCrtAsyncHttpClientStabilityTest extends S3BaseStabilityTest { - private static String bucketName = "s3crtasyncstabilitytests" + System.currentTimeMillis(); + private static String bucketName = "s3withcrtasyncclientstabilitytests" + System.currentTimeMillis(); private static S3AsyncClient s3CrtClient; @@ -33,6 +33,10 @@ public class S3CrtAsyncStabilityTest extends S3BaseStabilityTest { .build(); } + public S3WithCrtAsyncHttpClientStabilityTest() { + super(s3CrtClient); + } + @BeforeAll public static void setup() { s3CrtClient.createBucket(b -> b.bucket(bucketName)).join(); @@ -44,24 +48,9 @@ public static void cleanup() { s3CrtClient.close(); } - @Override - protected S3AsyncClient getTestClient() { return s3CrtClient; } - @Override protected String getTestBucketName() { return bucketName; } - @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class) - public void putObject_getObject_highConcurrency() { - putObject(); - getObject(); - } - - @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class) - public void largeObject_put_get_usingFile() { - uploadLargeObjectFromFile(); - downloadLargeObjectToFile(); - } - @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class) public void getBucketAcl_lowTpsLongInterval_Crt() { doGetBucketAcl_lowTpsLongInterval(); diff --git a/test/tests-coverage-reporting/pom.xml b/test/tests-coverage-reporting/pom.xml index e30d2cbd7764..f3e9b3a132ca 100644 --- a/test/tests-coverage-reporting/pom.xml +++ b/test/tests-coverage-reporting/pom.xml @@ -216,6 +216,11 @@ software.amazon.awssdk ${awsjavasdk.version} + + s3-transfer-manager + software.amazon.awssdk + ${awsjavasdk.version}-PREVIEW + diff --git a/utils/src/main/java/software/amazon/awssdk/utils/CompletableFutureUtils.java b/utils/src/main/java/software/amazon/awssdk/utils/CompletableFutureUtils.java index e679b9e9fe60..05f1c600d21f 100644 --- a/utils/src/main/java/software/amazon/awssdk/utils/CompletableFutureUtils.java +++ b/utils/src/main/java/software/amazon/awssdk/utils/CompletableFutureUtils.java @@ -17,6 +17,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.Executor; +import java.util.function.Function; import software.amazon.awssdk.annotations.SdkProtectedApi; /** @@ -76,4 +78,48 @@ public static CompletableFuture forwardExceptionTo(CompletableFuture s }); return src; } + + + /** + * Forward the {@code Throwable} that can be transformed as per the transformationFunction + * from {@code src} to {@code dst}. + * @param src The source of the {@code Throwable}. + * @param dst The destination where the {@code Throwable} will be forwarded to + * @param transformationFunction Transformation function taht will be applied on to the forwarded exception. + * @return + */ + public static CompletableFuture forwardTransformedExceptionTo(CompletableFuture src, + CompletableFuture dst, + Function + transformationFunction) { + src.whenComplete((r, e) -> { + if (e != null) { + dst.completeExceptionally(transformationFunction.apply(e)); + } + }); + return src; + } + + /** + * Completes the {@code dst} future based on the result of the {@code src} future asynchronously on + * the provided {@link Executor} and return the {@code src} future. + * + * @param src The source {@link CompletableFuture} + * @param dst The destination where the {@code Throwable} or response will be forwarded to. + * @param executor the executor to complete the des future + * @return the {@code src} future. + */ + public static CompletableFuture forwardResultTo(CompletableFuture src, + CompletableFuture dst, + Executor executor) { + src.whenCompleteAsync((r, e) -> { + if (e != null) { + dst.completeExceptionally(e); + } else { + dst.complete(r); + } + }, executor); + + return src; + } } diff --git a/utils/src/main/java/software/amazon/awssdk/utils/Validate.java b/utils/src/main/java/software/amazon/awssdk/utils/Validate.java index 7d98cc31246e..369ebf06968f 100644 --- a/utils/src/main/java/software/amazon/awssdk/utils/Validate.java +++ b/utils/src/main/java/software/amazon/awssdk/utils/Validate.java @@ -619,6 +619,13 @@ public static long isPositive(long num, String fieldName) { return num; } + public static double isPositive(double num, String fieldName) { + if (num <= 0) { + throw new IllegalArgumentException(String.format("%s must be positive", fieldName)); + } + return num; + } + public static int isNotNegative(int num, String fieldName) { if (num < 0) { @@ -685,6 +692,21 @@ public static Integer isPositiveOrNull(Integer num, String fieldName) { return isPositive(num, fieldName); } + /** + * Asserts that the given boxed double is positive (non-negative and non-zero) or null. + * + * @param num Boxed double to validate + * @param fieldName Field name to display in exception message if not positive. + * @return Duration if double or null. + */ + public static Double isPositiveOrNull(Double num, String fieldName) { + if (num == null) { + return null; + } + + return isPositive(num, fieldName); + } + /** * Asserts that the given boxed long is positive (non-negative and non-zero) or null. * diff --git a/utils/src/test/java/software/amazon/awssdk/utils/CompletableFutureUtilsTest.java b/utils/src/test/java/software/amazon/awssdk/utils/CompletableFutureUtilsTest.java index 14bea8844dc3..e2756cbbff66 100644 --- a/utils/src/test/java/software/amazon/awssdk/utils/CompletableFutureUtilsTest.java +++ b/utils/src/test/java/software/amazon/awssdk/utils/CompletableFutureUtilsTest.java @@ -15,14 +15,30 @@ package software.amazon.awssdk.utils; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.Assert.fail; public class CompletableFutureUtilsTest { + private static ExecutorService executors; + + @BeforeClass + public static void setup() { + executors = Executors.newFixedThreadPool(2); + } + + @AfterClass + public static void tearDown() { + executors.shutdown(); + } @Test(timeout = 1000) public void testForwardException() { @@ -42,4 +58,28 @@ public void testForwardException() { assertThat(t.getCause()).isEqualTo(e); } } + + @Test(timeout = 1000) + public void forwardResultTo_srcCompletesSuccessfully_shouldCompleteDstFuture() { + CompletableFuture src = new CompletableFuture<>(); + CompletableFuture dst = new CompletableFuture<>(); + + CompletableFuture returnedFuture = CompletableFutureUtils.forwardResultTo(src, dst, executors); + assertThat(returnedFuture).isEqualTo(src); + + src.complete("foobar"); + assertThat(dst.join()).isEqualTo("foobar"); + } + + @Test(timeout = 1000) + public void forwardResultTo_srcCompletesExceptionally_shouldCompleteDstFuture() { + CompletableFuture src = new CompletableFuture<>(); + CompletableFuture dst = new CompletableFuture<>(); + + RuntimeException exception = new RuntimeException("foobar"); + CompletableFutureUtils.forwardResultTo(src, dst, executors); + + src.completeExceptionally(exception); + assertThatThrownBy(dst::join).hasCause(exception); + } } diff --git a/utils/src/test/java/software/amazon/awssdk/utils/ValidateTest.java b/utils/src/test/java/software/amazon/awssdk/utils/ValidateTest.java index 261e3032df31..2983398f83d9 100644 --- a/utils/src/test/java/software/amazon/awssdk/utils/ValidateTest.java +++ b/utils/src/test/java/software/amazon/awssdk/utils/ValidateTest.java @@ -578,6 +578,31 @@ public void isPositiveOrNullLong_negative_throws() { Validate.isPositiveOrNull(-1L, "foo"); } + @Test + public void isPositiveOrNullDouble_null_returnsNull() { + assertNull(Validate.isPositiveOrNull((Double) null, "foo")); + } + + @Test + public void isPositiveOrNullDouble_positive_returnsInteger() { + Double num = 100.0; + assertEquals(num, Validate.isPositiveOrNull(num, "foo")); + } + + @Test + public void isPositiveOrNullDouble_zero_throws() { + expected.expect(IllegalArgumentException.class); + expected.expectMessage("foo"); + Validate.isPositiveOrNull(0.0, "foo"); + } + + @Test + public void isPositiveOrNullDouble_negative_throws() { + expected.expect(IllegalArgumentException.class); + expected.expectMessage("foo"); + Validate.isPositiveOrNull(-1.0, "foo"); + } + @Test public void isNull_notNull_shouldThrow() { expected.expect(IllegalArgumentException.class);