Improve availability of instance profile credentials provider during outages. (#2989)

This is done by allowing services to validate credential expiration time during IMDS outages instead of assuming the SDK has access to the latest valid expiration time.

Additional changes:
1. Removed unnecessary intellij inspections that have frequent false positives.
2. Include SDK user-agent in container credential provider calls.
3. Allow specifying the profile file and name used by the instance profile credentials provider.
4. Use the client's profile file and name for instance profile credentials when the default credentials provider is not overridden.
5. Convert the HttpCredentialsProvider protected API to a public API (with some breaking changes), so that there's fewer public classes extending internal classes.

Author: millems

.changes/next-release/bugfix-AWSSDKforJavav2-2a8f703.json

@@ -0,0 +1,6 @@
+{
+    "category": "AWS SDK for Java v2", 
+    "contributor": "", 
+    "type": "bugfix", 
+    "description": "Moved HttpCredentialsProvider (base class of ContainerCredentialsProvider and InstanceProfileCredentialsProvider) from private to public. This fixes an issue where public classes extended an internal class. Some components of this type were modified to allow it to be public."
+}

.changes/next-release/feature-AWSSDKforJavav2-b9ae28c.json

@@ -0,0 +1,6 @@
+{
+    "category": "AWS SDK for Java v2", 
+    "contributor": "", 
+    "type": "feature", 
+    "description": "Include SDK user-agent in container credential provider calls."
+}

.changes/next-release/feature-AWSSDKforJavav2-bc22481.json

@@ -0,0 +1,6 @@
+{
+    "category": "AWS SDK for Java v2", 
+    "contributor": "", 
+    "type": "feature", 
+    "description": "Allow specifying the profile file and name used by the instance profile credentials provider."
+}

.changes/next-release/feature-AWSSDKforJavav2-eca7ddc.json

@@ -0,0 +1,6 @@
+{
+    "category": "AWS SDK for Java v2", 
+    "contributor": "", 
+    "type": "feature", 
+    "description": "Improve resilience of instance profile credentials provider to short-term outages. Credentials that are close to expiration or expired can still be used to sign calls when the instance metadata service appears to be having issues. Services are now responsible for determining whether the credentials have actually expired."
+}

.changes/next-release/feature-AWSSDKforJavav2-f9ce53c.json

@@ -0,0 +1,6 @@
+{
+    "category": "AWS SDK for Java v2", 
+    "contributor": "", 
+    "type": "feature", 
+    "description": "Use the client's profile file and name for instance profile credentials when the default credentials provider is not overridden."
+}

.idea/inspectionProfiles/AWS_Java_SDK_2_0.xml

@@ -15,25 +15,33 @@
 
 package software.amazon.awssdk.auth.credentials;
 
-import static java.util.Collections.singletonMap;
 import static java.util.Collections.unmodifiableSet;
 
 import java.io.IOException;
 import java.net.URI;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
-import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import software.amazon.awssdk.annotations.SdkPublicApi;
-import software.amazon.awssdk.annotations.SdkTestInternalApi;
 import software.amazon.awssdk.auth.credentials.internal.ContainerCredentialsRetryPolicy;
+import software.amazon.awssdk.auth.credentials.internal.HttpCredentialsLoader;
+import software.amazon.awssdk.auth.credentials.internal.HttpCredentialsLoader.LoadedCredentials;
 import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.core.util.SdkUserAgent;
 import software.amazon.awssdk.regions.util.ResourcesEndpointProvider;
 import software.amazon.awssdk.regions.util.ResourcesEndpointRetryPolicy;
+import software.amazon.awssdk.utils.ComparableUtils;
 import software.amazon.awssdk.utils.StringUtils;
 import software.amazon.awssdk.utils.ToString;
+import software.amazon.awssdk.utils.Validate;
+import software.amazon.awssdk.utils.cache.CachedSupplier;
+import software.amazon.awssdk.utils.cache.NonBlocking;
+import software.amazon.awssdk.utils.cache.RefreshResult;
 
 /**
  * {@link AwsCredentialsProvider} implementation that loads credentials from a local metadata service.
@@ -52,15 +60,28 @@
  * Service (ECS)</a>
  */
 @SdkPublicApi
-public final class ContainerCredentialsProvider extends HttpCredentialsProvider {
-    private final ResourcesEndpointProvider credentialsEndpointProvider;
+public final class ContainerCredentialsProvider implements HttpCredentialsProvider {
+    private static final Set<String> ALLOWED_HOSTS = unmodifiableSet(new HashSet<>(Arrays.asList("localhost", "127.0.0.1")));
+
+    private final String endpoint;
+    private final HttpCredentialsLoader httpCredentialsLoader;
+    private final CachedSupplier<AwsCredentials> credentialsCache;
 
     /**
      * @see #builder()
      */
     private ContainerCredentialsProvider(BuilderImpl builder) {
-        super(builder);
-        this.credentialsEndpointProvider = builder.credentialsEndpointProvider;
+        this.endpoint = builder.endpoint;
+        this.httpCredentialsLoader = HttpCredentialsLoader.create();
+
+        if (Boolean.TRUE.equals(builder.asyncCredentialUpdateEnabled)) {
+            Validate.paramNotBlank(builder.asyncThreadName, "asyncThreadName");
+            this.credentialsCache = CachedSupplier.builder(this::refreshCredentials)
+                                                  .prefetchStrategy(new NonBlocking(builder.asyncThreadName))
+                                                  .build();
+        } else {
+            this.credentialsCache = CachedSupplier.builder(this::refreshCredentials).build();
+        }
     }
 
     /**
@@ -71,21 +92,60 @@ public static Builder builder() {
     }
 
     @Override
-    protected ResourcesEndpointProvider getCredentialsEndpointProvider() {
-        return credentialsEndpointProvider;
+    public String toString() {
+        return ToString.create("ContainerCredentialsProvider");
+    }
+
+    private RefreshResult<AwsCredentials> refreshCredentials() {
+        LoadedCredentials loadedCredentials =
+            httpCredentialsLoader.loadCredentials(new ContainerCredentialsEndpointProvider(endpoint));
+        Instant expiration = loadedCredentials.getExpiration().orElse(null);
+
+        return RefreshResult.builder(loadedCredentials.getAwsCredentials())
+                            .staleTime(staleTime(expiration))
+                            .prefetchTime(prefetchTime(expiration))
+                            .build();
+    }
+
+    private Instant staleTime(Instant expiration) {
+        if (expiration == null) {
+            return null;
+        }
+
+        return expiration.minus(1, ChronoUnit.MINUTES);
+    }
+
+    private Instant prefetchTime(Instant expiration) {
+        Instant oneHourFromNow = Instant.now().plus(1, ChronoUnit.HOURS);
+
+        if (expiration == null) {
+            return oneHourFromNow;
+        }
+
+        Instant fifteenMinutesBeforeExpiration = expiration.minus(15, ChronoUnit.MINUTES);
+
+        return ComparableUtils.minimum(oneHourFromNow, fifteenMinutesBeforeExpiration);
     }
 
     @Override
-    public String toString() {
-        return ToString.create("ContainerCredentialsProvider");
+    public AwsCredentials resolveCredentials() {
+        return credentialsCache.get();
+    }
+
+    @Override
+    public void close() {
+        credentialsCache.close();
     }
 
     static final class ContainerCredentialsEndpointProvider implements ResourcesEndpointProvider {
-        private static final Set<String> ALLOWED_HOSTS = unmodifiableSet(new HashSet<>(Arrays.asList("localhost", "127.0.0.1")));
+        private final String endpoint;
+
+        ContainerCredentialsEndpointProvider(String endpoint) {
+            this.endpoint = endpoint;
+        }
 
         @Override
         public URI endpoint() throws IOException {
-
             if (!SdkSystemSetting.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI.getStringValue().isPresent() &&
                 !SdkSystemSetting.AWS_CONTAINER_CREDENTIALS_FULL_URI.getStringValue().isPresent()) {
                 throw SdkClientException.builder()
@@ -117,26 +177,28 @@ public ResourcesEndpointRetryPolicy retryPolicy() {
 
         @Override
         public Map<String, String> headers() {
-            return SdkSystemSetting.AWS_CONTAINER_AUTHORIZATION_TOKEN.getStringValue()
-                                                                     .filter(StringUtils::isNotBlank)
-                                                                     .map(t -> singletonMap("Authorization", t))
-                                                                     .orElseGet(Collections::emptyMap);
+            Map<String, String> requestHeaders = new HashMap<>();
+            requestHeaders.put("User-Agent", SdkUserAgent.create().userAgent());
+            SdkSystemSetting.AWS_CONTAINER_AUTHORIZATION_TOKEN.getStringValue()
+                                                              .filter(StringUtils::isNotBlank)
+                                                              .ifPresent(t -> requestHeaders.put("Authorization", t));
+            return requestHeaders;
         }
 
         private URI createUri(String relativeUri) {
-            return URI.create(SdkSystemSetting.AWS_CONTAINER_SERVICE_ENDPOINT.getStringValueOrThrow() + relativeUri);
+            String host = endpoint != null ? endpoint : SdkSystemSetting.AWS_CONTAINER_SERVICE_ENDPOINT.getStringValueOrThrow();
+            return URI.create(host + relativeUri);
         }
 
         private URI createGenericContainerUrl() {
             URI uri = URI.create(SdkSystemSetting.AWS_CONTAINER_CREDENTIALS_FULL_URI.getStringValueOrThrow());
             if (!ALLOWED_HOSTS.contains(uri.getHost())) {
-
+                String envVarName = SdkSystemSetting.AWS_CONTAINER_CREDENTIALS_FULL_URI.environmentVariable();
                 throw SdkClientException.builder()
                                         .message(String.format("The full URI (%s) contained within environment " +
-                                                 "variable %s has an invalid host. Host can only be one of [%s].",
-                                                 uri,
-                                                 SdkSystemSetting.AWS_CONTAINER_CREDENTIALS_FULL_URI
-                                                                 .environmentVariable(),
+                                                               "variable %s has an invalid host. Host can only be one of [%s].",
+                                                               uri,
+                                                               envVarName,
                                                                String.join(",", ALLOWED_HOSTS)))
                                         .build();
             }
@@ -148,29 +210,47 @@ private URI createGenericContainerUrl() {
      * A builder for creating a custom a {@link ContainerCredentialsProvider}.
      */
     public interface Builder extends HttpCredentialsProvider.Builder<ContainerCredentialsProvider, Builder> {
+    }
+
+    private static final class BuilderImpl implements Builder {
+        private String endpoint;
+        private Boolean asyncCredentialUpdateEnabled;
+        private String asyncThreadName;
+
+        BuilderImpl() {
+            asyncThreadName("container-credentials-provider");
+        }
 
-        /**
-         * Build a {@link ContainerCredentialsProvider} from the provided configuration.
-         */
         @Override
-        ContainerCredentialsProvider build();
-    }
+        public Builder endpoint(String endpoint) {
+            this.endpoint = endpoint;
+            return this;
+        }
 
-    static final class BuilderImpl extends HttpCredentialsProvider.BuilderImpl<ContainerCredentialsProvider, Builder>
-        implements Builder {
+        public void setEndpoint(String endpoint) {
+            endpoint(endpoint);
+        }
 
-        private ResourcesEndpointProvider credentialsEndpointProvider = new ContainerCredentialsEndpointProvider();
+        @Override
+        public Builder asyncCredentialUpdateEnabled(Boolean asyncCredentialUpdateEnabled) {
+            this.asyncCredentialUpdateEnabled = asyncCredentialUpdateEnabled;
+            return this;
+        }
 
-        BuilderImpl() {
-            super.asyncThreadName("container-credentials-provider");
+        public void setAsyncCredentialUpdateEnabled(boolean asyncCredentialUpdateEnabled) {
+            asyncCredentialUpdateEnabled(asyncCredentialUpdateEnabled);
         }
 
-        @SdkTestInternalApi
-        Builder credentialsEndpointProvider(ResourcesEndpointProvider credentialsEndpointProvider) {
-            this.credentialsEndpointProvider = credentialsEndpointProvider;
+        @Override
+        public Builder asyncThreadName(String asyncThreadName) {
+            this.asyncThreadName = asyncThreadName;
             return this;
         }
 
+        public void setAsyncThreadName(String asyncThreadName) {
+            asyncThreadName(asyncThreadName);
+        }
+
         @Override
         public ContainerCredentialsProvider build() {
             return new ContainerCredentialsProvider(this);

core/auth/src/main/java/software/amazon/awssdk/auth/credentials/ContainerCredentialsProvider.java

@@ -83,6 +83,8 @@ private static LazyAwsCredentialsProvider createChain(Builder builder) {
                                                 .build(),
                     InstanceProfileCredentialsProvider.builder()
                                                       .asyncCredentialUpdateEnabled(asyncCredentialUpdateEnabled)
+                                                      .profileFile(builder.profileFile)
+                                                      .profileName(builder.profileName)
                                                       .build()
             };