Merge transfer manager change into mainline (#2649)

* Add download implementation for API requests

* Add S3CrtAsyncClient skeleton

* Add support for adapting the sdk configurations to crt configurations.

* Implement getObject in S3CrtAsyncClient

* Implement S3 CRT upload

* Fix checkstyle errors, add stability tests

* Upgrade crt to 0.11.5

* Update APIs

* Fix CRT S3 Client builder

 - Return the correct type from builder methods
 - Make max throughput and part size nullable

* Implement S3TransferManager#upload

* Implement TransferManager download and add s3-benchmarks module

* Move s3 crt related classes to s3-transfermanager

* Add requestLogger and requestIdLogger

* Various fixes and refactoring

* Add user-agent header in transfer manager

* Refactor UserAgentUtils and make it a protected API

* Fix sdk-benchmarks version

* Rename module and package for transfermanager

* Fix stability tests to not create a new client per request

* Various renaming

* Remove the option to pass bucket and key to DownloadRequest and UploadRequest and remove unused files

* Add README for S3TransferManager

* Add validation in S3ClientConfiguration

* Fix CRT/SDK POJO conversion and add more tests

* Update to use CRT DelegateCredentialsProvider

* Provide HTTP request-level data to PutObjectResponse

* Update Javadocs

* Fix the race condition in the FileAsyncRequestBody which causes the request to hang

* Add request cancellation logic and tests

* Fixed an issue where the future gets stuck when upload fails

* Offload completion of execution future to a separate thread

* Wrap CrtS3RuntimeException to SdkServiceException

* Revert "Wrap CrtS3RuntimeException to SdkServiceException"

* Update s3 crt client tests to verify resources are closed properly

* Wrap CrtS3RuntimeException to S3ServiceException

* Update version

* Bump up CRT version, clean up resources in tests, fix credential adapter TODO (#2594)

* Remove buffer copy since it's no longer needed (#2611)

* Fix fd remaining open longer than necessary (#2616)

In FileAsyncRequestBody, the file was not being closed until a request()
read past the end of the file without ready any bytes; i.e. if a read()
call reads up to the end of the file, it would not complete the
subscription on the subsequent request() signal. This fixes this issue
by signaling complete if we know we've reached the end.

* Cancel sub on finish and exception (#2619)

* Bump up crt version (#2640)

Bump up crt version

* Fix merge conflict

* Merge 'origin/master' into transfer manager branch (#2647)

Co-authored-by: Quanzzzz <[email protected]>
Co-authored-by: John Viegas <[email protected]>
Co-authored-by: Dongie Agnir <[email protected]>

Author: 4 people

.changes/next-release/bugfix-AWSSDKforJavav2-0da2191.json

@@ -0,0 +1,6 @@
+{
+    "type": "bugfix",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Fix issue in `FileAsyncRequestBody` where the underlying file channel would only be closed when enough requests are sent to read *past* the end of the file; if just enough requests are sent to read to the end of the file, the file is not closed, leaving an open file descriptor around longer than it needs to be."
+}

.changes/next-release/feature-AmazonS3TransferManagerPreview-e951466.json

@@ -0,0 +1,6 @@
+{
+    "category": "Amazon S3 Transfer Manager [Preview]", 
+    "contributor": "", 
+    "type": "feature", 
+    "description": "This release includes the preview release of the Amazon S3 Transfer Manager. Visit our [Developer Guide](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/transfer-manager.html) for more information."
+}

buildspecs/release-to-maven.yml

@@ -26,7 +26,7 @@ phases:
       if ! curl -f --head $SONATYPE_URL; then
         mkdir -p $CREDENTIALS
         aws s3 cp s3://aws-java-sdk-release-credentials/ $CREDENTIALS/ --recursive
-        mvn clean deploy -B -s $SETTINGS_XML -Dgpg.homedir=$GPG_HOME -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-sts-testing -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30
+        mvn clean deploy -B -s $SETTINGS_XML -Dgpg.homedir=$GPG_HOME -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-sts-testing,!:s3-benchmarks -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30
       else
         echo "This version was already released."
       fi

core/auth/src/main/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProvider.java

@@ -24,7 +24,7 @@
 import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.exception.SdkServiceException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.HttpResourcesUtils;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 import software.amazon.awssdk.utils.ToString;
@@ -160,7 +160,7 @@ public URI endpoint() throws IOException {
         @Override
         public Map<String, String> headers() {
             Map<String, String> requestHeaders = new HashMap<>();
-            requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+            requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
             requestHeaders.put("Accept", "*/*");
             requestHeaders.put("Connection", "keep-alive");
 
@@ -195,7 +195,7 @@ public URI endpoint() {
         @Override
         public Map<String, String> headers() {
             Map<String, String> requestHeaders = new HashMap<>();
-            requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+            requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
             requestHeaders.put("Accept", "*/*");
             requestHeaders.put("Connection", "keep-alive");
             requestHeaders.put(EC2_METADATA_TOKEN_TTL_HEADER, DEFAULT_TOKEN_TTL);

core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ContainerCredentialsProviderTest.java

@@ -33,7 +33,7 @@
 import org.junit.ClassRule;
 import org.junit.Test;
 import software.amazon.awssdk.core.exception.SdkClientException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 import software.amazon.awssdk.testutils.EnvironmentVariableHelper;
 
@@ -122,7 +122,7 @@ private void stubForCorruptedSuccessResponse() {
 
     private void stubFor200Response(String body) {
         stubFor(get(urlPathEqualTo(CREDENTIALS_PATH))
-                        .withHeader("User-Agent", equalTo(UserAgentUtils.getUserAgent()))
+                        .withHeader("User-Agent", equalTo(SdkUserAgent.create().userAgent()))
                         .willReturn(aResponse()
                                             .withStatus(200)
                                             .withHeader("Content-Type", "application/json")

core/auth/src/test/java/software/amazon/awssdk/auth/credentials/HttpCredentialsProviderTest.java

@@ -35,7 +35,7 @@
 import org.junit.ClassRule;
 import org.junit.Test;
 import software.amazon.awssdk.core.exception.SdkClientException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 import software.amazon.awssdk.utils.DateUtils;
 import software.amazon.awssdk.utils.IoUtils;
@@ -132,7 +132,7 @@ public void basicCachingFunctionalityWorks() {
     private void stubForSuccessResponseWithCustomBody(String body) {
         stubFor(
             get(urlPathEqualTo(CREDENTIALS_PATH))
-                .withHeader("User-Agent", equalTo(UserAgentUtils.getUserAgent()))
+                .withHeader("User-Agent", equalTo(SdkUserAgent.create().userAgent()))
                 .willReturn(aResponse()
                                 .withStatus(200)
                                 .withHeader("Content-Type", "application/json")

core/auth/src/test/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProviderTest.java

@@ -23,14 +23,11 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.hamcrest.Matchers.instanceOf;
 import com.github.tomakehurst.wiremock.WireMockServer;
 import com.github.tomakehurst.wiremock.client.WireMock;
 import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
 import com.github.tomakehurst.wiremock.junit.WireMockRule;
 import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder;
-import java.net.SocketTimeoutException;
 import java.time.Duration;
 import java.time.Instant;
 import org.junit.AfterClass;
@@ -40,8 +37,7 @@
 import org.junit.rules.ExpectedException;
 import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.exception.SdkClientException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
-import software.amazon.awssdk.profiles.ProfileFileSystemSetting;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.utils.DateUtils;
 
 public class InstanceProfileCredentialsProviderTest {
@@ -94,7 +90,7 @@ public void resolveCredentials_requestsIncludeUserAgent() {
         provider.resolveCredentials();
 
         String userAgentHeader = "User-Agent";
-        String userAgent = UserAgentUtils.getUserAgent();
+        String userAgent = SdkUserAgent.create().userAgent();
         WireMock.verify(putRequestedFor(urlPathEqualTo(TOKEN_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent)));
         WireMock.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent)));
         WireMock.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH + "some-profile")).withHeader(userAgentHeader, equalTo(userAgent)));
@@ -235,7 +231,7 @@ public void resolveCredentials_customProfileFileAndName_usesCorrectEndpoint() {
             provider.resolveCredentials();
 
             String userAgentHeader = "User-Agent";
-            String userAgent = UserAgentUtils.getUserAgent();
+            String userAgent = SdkUserAgent.create().userAgent();
             mockMetadataEndpoint_2.verify(putRequestedFor(urlPathEqualTo(TOKEN_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent)));
             mockMetadataEndpoint_2.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent)));
             mockMetadataEndpoint_2.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH + "some-profile")).withHeader(userAgentHeader, equalTo(userAgent)));

core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java

@@ -32,7 +32,7 @@
 import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.exception.SdkServiceException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.protocols.jsoncore.JsonNode;
 import software.amazon.awssdk.protocols.jsoncore.JsonNodeParser;
 import software.amazon.awssdk.regions.util.HttpResourcesUtils;
@@ -633,7 +633,7 @@ public URI endpoint() {
         @Override
         public Map<String, String> headers() {
             Map<String, String> requestHeaders = new HashMap<>();
-            requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+            requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
             requestHeaders.put("Accept", "*/*");
             requestHeaders.put("Connection", "keep-alive");
 

core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/InstanceProviderTokenEndpointProvider.java

@@ -19,7 +19,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import software.amazon.awssdk.annotations.SdkInternalApi;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 
 @SdkInternalApi
@@ -43,7 +43,7 @@ public URI endpoint() {
     @Override
     public Map<String, String> headers() {
         Map<String, String> requestHeaders = new HashMap<>();
-        requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+        requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
         requestHeaders.put("Accept", "*/*");
         requestHeaders.put("Connection", "keep-alive");
         requestHeaders.put(EC2_METADATA_TOKEN_TTL_HEADER, DEFAULT_TOKEN_TTL);

core/regions/src/main/java/software/amazon/awssdk/regions/util/ResourcesEndpointProvider.java

@@ -20,7 +20,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import software.amazon.awssdk.annotations.SdkProtectedApi;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 
 /**
  * <p>
@@ -57,7 +57,7 @@ default ResourcesEndpointRetryPolicy retryPolicy() {
      */
     default Map<String, String> headers() {
         Map<String, String> requestHeaders = new HashMap<>();
-        requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+        requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
         requestHeaders.put("Accept", "*/*");
         requestHeaders.put("Connection", "keep-alive");
 

core/regions/src/test/java/software/amazon/awssdk/regions/util/HttpCredentialsUtilsTest.java

@@ -36,8 +36,7 @@
 import org.mockito.runners.MockitoJUnitRunner;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.exception.SdkServiceException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
-import software.amazon.awssdk.core.util.VersionInfo;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.internal.util.ConnectionUtils;
 import software.amazon.awssdk.regions.internal.util.SocketUtils;
 
@@ -53,7 +52,7 @@ public class HttpCredentialsUtilsTest {
     private static Map<String, String> headers = new HashMap<String, String>()
     {
         {
-            put("User-Agent", UserAgentUtils.getUserAgent());
+            put("User-Agent", SdkUserAgent.create().userAgent());
             put("Accept", "*/*");
             put("Connection", "keep-alive");
         }

core/sdk-core/src/main/java/software/amazon/awssdk/core/client/builder/SdkDefaultClientBuilder.java

@@ -63,9 +63,9 @@
 import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
 import software.amazon.awssdk.core.internal.http.loader.DefaultSdkAsyncHttpClientBuilder;
 import software.amazon.awssdk.core.internal.http.loader.DefaultSdkHttpClientBuilder;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
 import software.amazon.awssdk.core.retry.RetryMode;
 import software.amazon.awssdk.core.retry.RetryPolicy;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.http.ExecutableHttpRequest;
 import software.amazon.awssdk.http.HttpExecuteRequest;
 import software.amazon.awssdk.http.SdkHttpClient;
@@ -205,7 +205,7 @@ private SdkClientConfiguration mergeGlobalDefaults(SdkClientConfiguration config
                                          .option(ADDITIONAL_HTTP_HEADERS, new LinkedHashMap<>())
                                          .option(PROFILE_FILE, profileFile)
                                          .option(PROFILE_NAME, ProfileFileSystemSetting.AWS_PROFILE.getStringValueOrThrow())
-                                         .option(USER_AGENT_PREFIX, UserAgentUtils.getUserAgent())
+                                         .option(USER_AGENT_PREFIX, SdkUserAgent.create().userAgent())
                                          .option(USER_AGENT_SUFFIX, "")
                                          .option(CRC32_FROM_COMPRESSED_DATA_ENABLED, false));
     }
@@ -298,6 +298,9 @@ protected AttributeMap childHttpConfig() {
      * Finalize which async executor service will be used for the created client. The default async executor
      * service has at least 8 core threads and can scale up to at least 64 threads when needed depending
      * on the number of processors available.
+     *
+     * This uses the same default executor in S3NativeClientConfiguration#resolveAsyncFutureCompletionExecutor.
+     * Make sure you update that method if you update the defaults here.
      */
     private Executor resolveAsyncFutureCompletionExecutor(SdkClientConfiguration config) {
         Supplier<Executor> defaultExecutor = () -> {