Merge transfer manager change into mainline

.changes/next-release/bugfix-AWSSDKforJavav2-0da2191.json

@@ -0,0 +1,6 @@
+{
+    "type": "bugfix",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Fix issue in `FileAsyncRequestBody` where the underlying file channel would only be closed when enough requests are sent to read *past* the end of the file; if just enough requests are sent to read to the end of the file, the file is not closed, leaving an open file descriptor around longer than it needs to be."
+}

buildspecs/release-to-maven.yml

@@ -26,7 +26,7 @@ phases:
       if ! curl -f --head $SONATYPE_URL; then
         mkdir -p $CREDENTIALS
         aws s3 cp s3://aws-java-sdk-release-credentials/ $CREDENTIALS/ --recursive
-        mvn clean deploy -B -s $SETTINGS_XML -Dgpg.homedir=$GPG_HOME -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-sts-testing -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30
+        mvn clean deploy -B -s $SETTINGS_XML -Dgpg.homedir=$GPG_HOME -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-sts-testing,!:s3-benchmarks -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30
       else
         echo "This version was already released."
       fi

core/auth/src/main/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProvider.java

@@ -24,7 +24,7 @@
 import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.exception.SdkServiceException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.HttpResourcesUtils;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 import software.amazon.awssdk.utils.ToString;
@@ -160,7 +160,7 @@ public URI endpoint() throws IOException {
         @Override
         public Map<String, String> headers() {
             Map<String, String> requestHeaders = new HashMap<>();
-            requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+            requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
             requestHeaders.put("Accept", "*/*");
             requestHeaders.put("Connection", "keep-alive");
 
@@ -195,7 +195,7 @@ public URI endpoint() {
         @Override
         public Map<String, String> headers() {
             Map<String, String> requestHeaders = new HashMap<>();
-            requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+            requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
             requestHeaders.put("Accept", "*/*");
             requestHeaders.put("Connection", "keep-alive");
             requestHeaders.put(EC2_METADATA_TOKEN_TTL_HEADER, DEFAULT_TOKEN_TTL);

core/auth/src/test/java/software/amazon/awssdk/auth/credentials/ContainerCredentialsProviderTest.java

@@ -33,7 +33,7 @@
 import org.junit.ClassRule;
 import org.junit.Test;
 import software.amazon.awssdk.core.exception.SdkClientException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 import software.amazon.awssdk.testutils.EnvironmentVariableHelper;
 
@@ -122,7 +122,7 @@ private void stubForCorruptedSuccessResponse() {
 
     private void stubFor200Response(String body) {
         stubFor(get(urlPathEqualTo(CREDENTIALS_PATH))
-                        .withHeader("User-Agent", equalTo(UserAgentUtils.getUserAgent()))
+                        .withHeader("User-Agent", equalTo(SdkUserAgent.create().userAgent()))
                         .willReturn(aResponse()
                                             .withStatus(200)
                                             .withHeader("Content-Type", "application/json")

core/auth/src/test/java/software/amazon/awssdk/auth/credentials/HttpCredentialsProviderTest.java

@@ -35,7 +35,7 @@
 import org.junit.ClassRule;
 import org.junit.Test;
 import software.amazon.awssdk.core.exception.SdkClientException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 import software.amazon.awssdk.utils.DateUtils;
 import software.amazon.awssdk.utils.IoUtils;
@@ -132,7 +132,7 @@ public void basicCachingFunctionalityWorks() {
     private void stubForSuccessResponseWithCustomBody(String body) {
         stubFor(
             get(urlPathEqualTo(CREDENTIALS_PATH))
-                .withHeader("User-Agent", equalTo(UserAgentUtils.getUserAgent()))
+                .withHeader("User-Agent", equalTo(SdkUserAgent.create().userAgent()))
                 .willReturn(aResponse()
                                 .withStatus(200)
                                 .withHeader("Content-Type", "application/json")

core/auth/src/test/java/software/amazon/awssdk/auth/credentials/InstanceProfileCredentialsProviderTest.java

@@ -23,14 +23,11 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.hamcrest.Matchers.instanceOf;
 import com.github.tomakehurst.wiremock.WireMockServer;
 import com.github.tomakehurst.wiremock.client.WireMock;
 import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
 import com.github.tomakehurst.wiremock.junit.WireMockRule;
 import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder;
-import java.net.SocketTimeoutException;
 import java.time.Duration;
 import java.time.Instant;
 import org.junit.AfterClass;
@@ -40,8 +37,7 @@
 import org.junit.rules.ExpectedException;
 import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.exception.SdkClientException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
-import software.amazon.awssdk.profiles.ProfileFileSystemSetting;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.utils.DateUtils;
 
 public class InstanceProfileCredentialsProviderTest {
@@ -94,7 +90,7 @@ public void resolveCredentials_requestsIncludeUserAgent() {
         provider.resolveCredentials();
 
         String userAgentHeader = "User-Agent";
-        String userAgent = UserAgentUtils.getUserAgent();
+        String userAgent = SdkUserAgent.create().userAgent();
         WireMock.verify(putRequestedFor(urlPathEqualTo(TOKEN_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent)));
         WireMock.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent)));
         WireMock.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH + "some-profile")).withHeader(userAgentHeader, equalTo(userAgent)));
@@ -235,7 +231,7 @@ public void resolveCredentials_customProfileFileAndName_usesCorrectEndpoint() {
             provider.resolveCredentials();
 
             String userAgentHeader = "User-Agent";
-            String userAgent = UserAgentUtils.getUserAgent();
+            String userAgent = SdkUserAgent.create().userAgent();
             mockMetadataEndpoint_2.verify(putRequestedFor(urlPathEqualTo(TOKEN_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent)));
             mockMetadataEndpoint_2.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH)).withHeader(userAgentHeader, equalTo(userAgent)));
             mockMetadataEndpoint_2.verify(getRequestedFor(urlPathEqualTo(CREDENTIALS_RESOURCE_PATH + "some-profile")).withHeader(userAgentHeader, equalTo(userAgent)));

core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java

@@ -32,7 +32,7 @@
 import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.exception.SdkServiceException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.protocols.jsoncore.JsonNode;
 import software.amazon.awssdk.protocols.jsoncore.JsonNodeParser;
 import software.amazon.awssdk.regions.util.HttpResourcesUtils;
@@ -633,7 +633,7 @@ public URI endpoint() {
         @Override
         public Map<String, String> headers() {
             Map<String, String> requestHeaders = new HashMap<>();
-            requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+            requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
             requestHeaders.put("Accept", "*/*");
             requestHeaders.put("Connection", "keep-alive");
 

core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/InstanceProviderTokenEndpointProvider.java

@@ -19,7 +19,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import software.amazon.awssdk.annotations.SdkInternalApi;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 
 @SdkInternalApi
@@ -43,7 +43,7 @@ public URI endpoint() {
     @Override
     public Map<String, String> headers() {
         Map<String, String> requestHeaders = new HashMap<>();
-        requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+        requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
         requestHeaders.put("Accept", "*/*");
         requestHeaders.put("Connection", "keep-alive");
         requestHeaders.put(EC2_METADATA_TOKEN_TTL_HEADER, DEFAULT_TOKEN_TTL);

core/regions/src/main/java/software/amazon/awssdk/regions/util/ResourcesEndpointProvider.java

@@ -20,7 +20,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import software.amazon.awssdk.annotations.SdkProtectedApi;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 
 /**
  * <p>
@@ -57,7 +57,7 @@ default ResourcesEndpointRetryPolicy retryPolicy() {
      */
     default Map<String, String> headers() {
         Map<String, String> requestHeaders = new HashMap<>();
-        requestHeaders.put("User-Agent", UserAgentUtils.getUserAgent());
+        requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
         requestHeaders.put("Accept", "*/*");
         requestHeaders.put("Connection", "keep-alive");
 

core/regions/src/test/java/software/amazon/awssdk/regions/util/HttpCredentialsUtilsTest.java

@@ -36,8 +36,7 @@
 import org.mockito.runners.MockitoJUnitRunner;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.exception.SdkServiceException;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
-import software.amazon.awssdk.core.util.VersionInfo;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.internal.util.ConnectionUtils;
 import software.amazon.awssdk.regions.internal.util.SocketUtils;
 
@@ -53,7 +52,7 @@ public class HttpCredentialsUtilsTest {
     private static Map<String, String> headers = new HashMap<String, String>()
     {
         {
-            put("User-Agent", UserAgentUtils.getUserAgent());
+            put("User-Agent", SdkUserAgent.create().userAgent());
             put("Accept", "*/*");
             put("Connection", "keep-alive");
         }

core/sdk-core/src/main/java/software/amazon/awssdk/core/client/builder/SdkDefaultClientBuilder.java

@@ -63,9 +63,9 @@
 import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
 import software.amazon.awssdk.core.internal.http.loader.DefaultSdkAsyncHttpClientBuilder;
 import software.amazon.awssdk.core.internal.http.loader.DefaultSdkHttpClientBuilder;
-import software.amazon.awssdk.core.internal.util.UserAgentUtils;
 import software.amazon.awssdk.core.retry.RetryMode;
 import software.amazon.awssdk.core.retry.RetryPolicy;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.http.ExecutableHttpRequest;
 import software.amazon.awssdk.http.HttpExecuteRequest;
 import software.amazon.awssdk.http.SdkHttpClient;
@@ -205,7 +205,7 @@ private SdkClientConfiguration mergeGlobalDefaults(SdkClientConfiguration config
                                          .option(ADDITIONAL_HTTP_HEADERS, new LinkedHashMap<>())
                                          .option(PROFILE_FILE, profileFile)
                                          .option(PROFILE_NAME, ProfileFileSystemSetting.AWS_PROFILE.getStringValueOrThrow())
-                                         .option(USER_AGENT_PREFIX, UserAgentUtils.getUserAgent())
+                                         .option(USER_AGENT_PREFIX, SdkUserAgent.create().userAgent())
                                          .option(USER_AGENT_SUFFIX, "")
                                          .option(CRC32_FROM_COMPRESSED_DATA_ENABLED, false));
     }
@@ -298,6 +298,9 @@ protected AttributeMap childHttpConfig() {
      * Finalize which async executor service will be used for the created client. The default async executor
      * service has at least 8 core threads and can scale up to at least 64 threads when needed depending
      * on the number of processors available.
+     *
+     * This uses the same default executor in S3NativeClientConfiguration#resolveAsyncFutureCompletionExecutor.
+     * Make sure you update that method if you update the defaults here.
      */
     private Executor resolveAsyncFutureCompletionExecutor(SdkClientConfiguration config) {
         Supplier<Executor> defaultExecutor = () -> {