Add specific http configurations for cloudwatch and s3

.changes/next-release/bugfix-AmazonCloudWatch-03b6e4e.json

@@ -0,0 +1,5 @@
+{
+    "category": "Amazon CloudWatch", 
+    "type": "bugfix", 
+    "description": "Add cloudwatch specific http configurations, specifically reducing `connectionMaxIdleTime`. Related to [#1380](https://github.com/aws/aws-sdk-java-v2/issues/1380)"
+}

.changes/next-release/bugfix-AmazonS3-2cc26c3.json

@@ -0,0 +1,5 @@
+{
+    "category": "Amazon S3", 
+    "type": "bugfix", 
+    "description": "Add s3 specific http configurations, specifically reducing `connectionMaxIdleTime`. Related to [#1122](https://github.com/aws/aws-sdk-java-v2/issues/1122)"
+}

services/cloudwatch/src/main/java/software/amazon/awssdk/services/cloudwatch/internal/CloudWatchHttpConfigurationOptions.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.cloudwatch.internal;
+
+import java.time.Duration;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.http.SdkHttpConfigurationOption;
+import software.amazon.awssdk.utils.AttributeMap;
+
+/**
+ * CloudWatch specific http configurations
+ */
+@SdkInternalApi
+public final class CloudWatchHttpConfigurationOptions {
+    private static final AttributeMap OPTIONS = AttributeMap
+        .builder()
+        .put(SdkHttpConfigurationOption.CONNECTION_MAX_IDLE_TIMEOUT, Duration.ofSeconds(5))
+        .build();
+
+    private CloudWatchHttpConfigurationOptions() {
+    }
+
+    public static AttributeMap defaultHttpConfig() {
+        return OPTIONS;
+    }
+}

services/cloudwatch/src/main/resources/codegen-resources/customization.config

@@ -9,5 +9,6 @@
     "deleteDashboards",
     "putDashboard",
     "getDashboard"
-  ]
+  ],
+  "serviceSpecificHttpConfig": "software.amazon.awssdk.services.cloudwatch.internal.CloudWatchHttpConfigurationOptions"
 }

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/S3HttpConfigurationOptions.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.internal;
+
+import java.time.Duration;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.http.SdkHttpConfigurationOption;
+import software.amazon.awssdk.utils.AttributeMap;
+
+/**
+ * S3 specific http configurations
+ */
+@SdkInternalApi
+public final class S3HttpConfigurationOptions {
+    private static final AttributeMap OPTIONS = AttributeMap
+        .builder()
+        .put(SdkHttpConfigurationOption.CONNECTION_MAX_IDLE_TIMEOUT, Duration.ofSeconds(5))
+        .build();
+
+    private S3HttpConfigurationOptions() {
+    }
+
+    public static AttributeMap defaultHttpConfig() {
+        return OPTIONS;
+    }
+}

services/s3/src/main/resources/codegen-resources/customization.config

@@ -85,5 +85,6 @@
     "createMethodParams": [
       "clientConfiguration"
     ]
-  }
+  },
+  "serviceSpecificHttpConfig": "software.amazon.awssdk.services.s3.internal.S3HttpConfigurationOptions"
 }

test/stability-tests/pom.xml

@@ -129,6 +129,12 @@
             <version>${awsjavasdk.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>cloudwatch</artifactId>
+            <version>${awsjavasdk.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>

test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/TestRunner.java

@@ -16,17 +16,22 @@
 package software.amazon.awssdk.stability.tests;
 
 
+/**
+ * The main method will be invoked when you execute the test jar generated from
+ * "mvn package -P test-jar"
+ *
+ * You can add the tests in the main method.
+ * eg:
+ *        try {
+ *            S3AsyncStabilityTest s3AsyncStabilityTest = new S3AsyncStabilityTest();
+ *            S3AsyncStabilityTest.setup();
+ *            s3AsyncStabilityTest.putObject_getObject();
+ *        } finally {
+ *            S3AsyncStabilityTest.cleanup();
+ *        }
+ */
 public class TestRunner {
 
     public static void main(String... args) {
-        // You can add the tests you want to run here.
-        // eg:
-//        try {
-//            S3AsyncStabilityTest s3AsyncStabilityTest = new S3AsyncStabilityTest();
-//            S3AsyncStabilityTest.setup();
-//            s3AsyncStabilityTest.putObject_getObject();
-//        } finally {
-//            S3AsyncStabilityTest.cleanup();
-//        }
     }
 }

test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/cloudwatch/CloudWatchAsyncStabilityTest.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.stability.tests.cloudwatch;
+
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.IntFunction;
+import org.apache.commons.lang3.RandomUtils;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import software.amazon.awssdk.services.cloudwatch.model.MetricDatum;
+import software.amazon.awssdk.stability.tests.exceptions.StabilityTestsRetryableException;
+import software.amazon.awssdk.stability.tests.utils.RetryableTest;
+import software.amazon.awssdk.stability.tests.utils.StabilityTestRunner;
+
+public class CloudWatchAsyncStabilityTest extends CloudWatchBaseStabilityTest {
+    private static String namespace;
+
+    @BeforeAll
+    public static void setup() {
+        namespace = "CloudWatchAsyncStabilityTest" + System.currentTimeMillis();
+    }
+
+    @AfterAll
+    public static void tearDown() {
+        cloudWatchAsyncClient.close();
+    }
+
+    @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class)
+    public void putMetrics_lowTpsLongInterval() {
+        List<MetricDatum> metrics = new ArrayList<>();
+        for (int i = 0; i < 20 ; i++) {
+            metrics.add(MetricDatum.builder()
+                                   .metricName("test")
+                                   .values(RandomUtils.nextDouble(1d, 1000d))
+                                   .build());
+        }
+
+        IntFunction<CompletableFuture<?>> futureIntFunction = i ->
+            cloudWatchAsyncClient.putMetricData(b -> b.namespace(namespace)
+                                                      .metricData(metrics));
+
+        runCloudWatchTest("putMetrics_lowTpsLongInterval", futureIntFunction);
+    }
+
+
+    private void runCloudWatchTest(String testName, IntFunction<CompletableFuture<?>> futureIntFunction) {
+        StabilityTestRunner.newRunner()
+                           .testName("CloudWatchAsyncStabilityTest." + testName)
+                           .futureFactory(futureIntFunction)
+                           .totalRuns(TOTAL_RUNS)
+                           .requestCountPerRun(CONCURRENCY)
+                           .delaysBetweenEachRun(Duration.ofSeconds(6))
+                           .run();
+    }
+}

test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/cloudwatch/CloudWatchBaseStabilityTest.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.stability.tests.cloudwatch;
+
+
+import java.time.Duration;
+import software.amazon.awssdk.core.retry.RetryPolicy;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
+import software.amazon.awssdk.testutils.service.AwsTestBase;
+
+public abstract class CloudWatchBaseStabilityTest extends AwsTestBase {
+    protected static final int CONCURRENCY = 50;
+    protected static final int TOTAL_RUNS = 3;
+
+    protected static CloudWatchAsyncClient cloudWatchAsyncClient =
+        CloudWatchAsyncClient.builder()
+                             .httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(CONCURRENCY))
+                             .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
+                             .overrideConfiguration(b -> b
+                                 // Retry at test level
+                                 .retryPolicy(RetryPolicy.none())
+                                 .apiCallTimeout(Duration.ofMinutes(1)))
+                             .build();
+
+
+}

test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3AsyncStabilityTest.java

@@ -61,6 +61,18 @@ public void largeObject_put_get_usingFile() {
         downloadLargeObjectToFile();
     }
 
+    @RetryableTest(maxRetries = 3, retryableException = StabilityTestsRetryableException.class)
+    public void getBucketAcl_lowTpsLongInterval() {
+        IntFunction<CompletableFuture<?>> future = i -> s3NettyClient.getBucketAcl(b -> b.bucket(bucketName));
+        StabilityTestRunner.newRunner()
+                           .testName("S3AsyncStabilityTest.getBucketAcl_lowTpsLongInterval")
+                           .futureFactory(future)
+                           .requestCountPerRun(10)
+                           .totalRuns(3)
+                           .delaysBetweenEachRun(Duration.ofSeconds(6))
+                           .run();
+    }
+
     private void downloadLargeObjectToFile() {
         File randomTempFile = RandomTempFile.randomUncreatedFile();
         StabilityTestRunner.newRunner()

test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/s3/S3BaseStabilityTest.java

@@ -20,6 +20,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import software.amazon.awssdk.core.retry.RetryPolicy;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.http.apache.ApacheHttpClient;
 import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
@@ -46,7 +47,9 @@ public abstract class S3BaseStabilityTest extends AwsTestBase {
                                      .httpClientBuilder(NettyNioAsyncHttpClient.builder()
                                                                                .maxConcurrency(CONCURRENCY))
                                      .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN)
-                                     .overrideConfiguration(b -> b.apiCallTimeout(Duration.ofMinutes(10)))
+                                     .overrideConfiguration(b -> b.apiCallTimeout(Duration.ofMinutes(10))
+                                                                  // Retry at test level
+                                                                  .retryPolicy(RetryPolicy.none()))
                                      .build();
 
 

test/stability-tests/src/it/java/software/amazon/awssdk/stability/tests/utils/StabilityTestRunner.java

@@ -251,7 +251,11 @@ private static CompletableFuture<?> handleException(CompletableFuture<?> future,
             } else if (isIOException(cause)) {
                 exceptionCounter.addIoException();
             } else if (cause instanceof SdkClientException) {
-                exceptionCounter.addClientException();
+                if (isIOException(cause.getCause())) {
+                    exceptionCounter.addIoException();
+                } else {
+                    exceptionCounter.addClientException();
+                }
             } else {
                 exceptionCounter.addUnknownException();
             }