Merge pull request #2820 from aws/feature/s3-express-release

Feature/s3 express release

Author: L-Applin

.changes/next-release/feature-AWSSDKforJavav2-10225f8.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Add `cloudwatch-metric-publisher` to `aws-sdk-java` module so that it can be included in the bundle JAR."
+}

.changes/next-release/feature-AmazonS3-010e379.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "Amazon S3",
+    "contributor": "",
+    "description": "Adds SDK support for Amazon S3 Express One Zone. For usage, consult Javadocs and the official documentation. Note that users can switch off CreateSession authentication and use the regular credential providers to sign requests by using the `disableS3ExpressSessionAuth` option on the client builder, environment variables, system properties or the config file."
+}

.gitignore

@@ -27,3 +27,5 @@ target/
 
 *.pyc
 *.swp
+
+

codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java

@@ -257,6 +257,8 @@ public class CustomizationConfig {
 
     private boolean useGlobalEndpoint;
 
+    private boolean useS3ExpressSessionAuth;
+
     private List<String> interceptors = new ArrayList<>();
 
     private List<String> internalPlugins = new ArrayList<>();
@@ -291,6 +293,13 @@ public class CustomizationConfig {
      */
     private Map<String, ClientContextParam> customClientContextParams;
 
+    private boolean s3ExpressAuthSupport;
+
+    /**
+     * Customization related to auth scheme derived from endpoints.
+     */
+    private EndpointAuthSchemeConfig endpointAuthSchemeConfig;
+
     private CustomizationConfig() {
     }
 
@@ -685,6 +694,14 @@ public void setUseGlobalEndpoint(boolean useGlobalEndpoint) {
         this.useGlobalEndpoint = useGlobalEndpoint;
     }
 
+    public boolean useS3ExpressSessionAuth() {
+        return useS3ExpressSessionAuth;
+    }
+
+    public void setUseS3ExpressSessionAuth(boolean useS3ExpressSessionAuth) {
+        this.useS3ExpressSessionAuth = useS3ExpressSessionAuth;
+    }
+
     public Map<String, String> getSkipEndpointTests() {
         return skipEndpointTests;
     }
@@ -756,11 +773,28 @@ public void setCustomClientContextParams(Map<String, ClientContextParam> customC
         this.customClientContextParams = customClientContextParams;
     }
 
+    public boolean getS3ExpressAuthSupport() {
+        return s3ExpressAuthSupport;
+    }
+
+    public void setS3ExpressAuthSupport(boolean s3ExpressAuthSupport) {
+        this.s3ExpressAuthSupport = s3ExpressAuthSupport;
+    }
+
     public MultipartCustomization getMultipartCustomization() {
         return this.multipartCustomization;
     }
 
     public void setMultipartCustomization(MultipartCustomization multipartCustomization) {
         this.multipartCustomization = multipartCustomization;
     }
+
+    public EndpointAuthSchemeConfig getEndpointAuthSchemeConfig() {
+        return endpointAuthSchemeConfig;
+    }
+
+    public void setEndpointAuthSchemeConfig(EndpointAuthSchemeConfig endpointAuthSchemeConfig) {
+        this.endpointAuthSchemeConfig = endpointAuthSchemeConfig;
+    }
+
 }

codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/EndpointAuthSchemeConfig.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.model.config.customization;
+
+import java.util.Map;
+
+public class EndpointAuthSchemeConfig {
+
+    private String authSchemeStrategyFactoryClass;
+    private String knownEndpointProperties;
+
+    private Map<String, KeyTypePair> endpointProviderTestKeys;
+
+    public String getAuthSchemeStrategyFactoryClass() {
+        return authSchemeStrategyFactoryClass;
+    }
+
+    public void setAuthSchemeStrategyFactoryClass(String authSchemeStrategyFactoryClass) {
+        this.authSchemeStrategyFactoryClass = authSchemeStrategyFactoryClass;
+    }
+
+    public String getKnownEndpointProperties() {
+        return knownEndpointProperties;
+    }
+
+    public void setKnownEndpointProperties(String knownEndpointProperties) {
+        this.knownEndpointProperties = knownEndpointProperties;
+    }
+
+    public Map<String, KeyTypePair> getEndpointProviderTestKeys() {
+        return endpointProviderTestKeys;
+    }
+
+    public void setEndpointProviderTestKeys(Map<String, KeyTypePair> endpointProviderTestKeys) {
+        this.endpointProviderTestKeys = endpointProviderTestKeys;
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/KeyTypePair.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.model.config.customization;
+
+import software.amazon.awssdk.utils.ToString;
+
+public class KeyTypePair {
+    private String key;
+    private String type;
+
+    public String getKey() {
+        return key;
+    }
+
+    public void setKey(String key) {
+        this.key = key;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    @Override
+    public String toString() {
+        return ToString.builder("KeyTypePair")
+                       .add("key", key)
+                       .add("type", type)
+                       .build();
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/model/rules/endpoints/ExpectModel.java

@@ -18,6 +18,7 @@
 import com.fasterxml.jackson.core.TreeNode;
 import java.util.List;
 import java.util.Map;
+import software.amazon.awssdk.utils.ToString;
 
 public class ExpectModel {
     private String error;
@@ -39,6 +40,14 @@ public void setEndpoint(Endpoint endpoint) {
         this.endpoint = endpoint;
     }
 
+    @Override
+    public String toString() {
+        return ToString.builder("ExpectModel")
+                       .add("error", error)
+                       .add("endpoint", endpoint)
+                       .build();
+    }
+
     public static class Endpoint {
         private String url;
         private Map<String, List<String>> headers;
@@ -67,5 +76,14 @@ public Map<String, TreeNode> getProperties() {
         public void setProperties(Map<String, TreeNode> properties) {
             this.properties = properties;
         }
+
+        @Override
+        public String toString() {
+            return ToString.builder("Endpoint")
+                           .add("url", url)
+                           .add("headers", headers)
+                           .add("properties", properties)
+                           .build();
+        }
     }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/poet/auth/scheme/AuthSchemeInterceptorSpec.java

@@ -18,6 +18,7 @@
 import com.squareup.javapoet.ClassName;
 import com.squareup.javapoet.FieldSpec;
 import com.squareup.javapoet.MethodSpec;
+import com.squareup.javapoet.ParameterSpec;
 import com.squareup.javapoet.ParameterizedTypeName;
 import com.squareup.javapoet.TypeName;
 import com.squareup.javapoet.TypeSpec;
@@ -40,6 +41,7 @@
 import software.amazon.awssdk.core.SdkRequest;
 import software.amazon.awssdk.core.SelectedAuthScheme;
 import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.identity.SdkIdentityProperty;
 import software.amazon.awssdk.core.interceptor.Context;
 import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
 import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
@@ -91,7 +93,8 @@ public TypeSpec poetSpec() {
                .addMethod(generateSelectAuthScheme())
                .addMethod(generateAuthSchemeParams())
                .addMethod(generateTrySelectAuthScheme())
-               .addMethod(generateGetIdentityMetric());
+               .addMethod(generateGetIdentityMetric())
+               .addMethod(putSelectedAuthSchemeMethodSpec());
         return builder.build();
     }
 
@@ -108,8 +111,7 @@ private MethodSpec generateBeforeExecution() {
                              listOf(AuthSchemeOption.class))
                .addStatement("$T selectedAuthScheme = selectAuthScheme(authOptions, executionAttributes)",
                              wildcardSelectedAuthScheme())
-               .addStatement("$T.putSelectedAuthScheme(executionAttributes, selectedAuthScheme)",
-                             endpointRulesSpecUtils.rulesRuntimeClassName("AuthSchemeUtils"));
+               .addStatement("putSelectedAuthScheme(executionAttributes, selectedAuthScheme)");
         return builder.build();
     }
 
@@ -208,7 +210,7 @@ private MethodSpec generateSelectAuthScheme() {
         {
             builder.addStatement("$T authScheme = authSchemes.get(authOption.schemeId())", wildcardAuthScheme())
                    .addStatement("$T selectedAuthScheme = trySelectAuthScheme(authOption, authScheme, identityProviders, "
-                                 + "discardedReasons, metricCollector)",
+                                 + "discardedReasons, metricCollector, executionAttributes)",
                                  wildcardSelectedAuthScheme());
             builder.beginControlFlow("if (selectedAuthScheme != null)");
             {
@@ -229,6 +231,7 @@ private MethodSpec generateSelectAuthScheme() {
         return builder.build();
     }
 
+    //TODO (s3express) Review "general" identity properties and their propagation
     private MethodSpec generateTrySelectAuthScheme() {
         MethodSpec.Builder builder = MethodSpec.methodBuilder("trySelectAuthScheme")
                                                .addModifiers(Modifier.PRIVATE)
@@ -238,6 +241,7 @@ private MethodSpec generateTrySelectAuthScheme() {
                                                .addParameter(IdentityProviders.class, "identityProviders")
                                                .addParameter(listOfStringSuppliers(), "discardedReasons")
                                                .addParameter(MetricCollector.class, "metricCollector")
+                                               .addParameter(ExecutionAttributes.class, "executionAttributes")
                                                .addTypeVariable(TypeVariableName.get("T", Identity.class));
 
         builder.beginControlFlow("if (authScheme == null)");
@@ -262,7 +266,12 @@ private MethodSpec generateTrySelectAuthScheme() {
                              ResolveIdentityRequest.class,
                              ResolveIdentityRequest.class);
         builder.addStatement("authOption.forEachIdentityProperty(identityRequestBuilder::putProperty)");
-
+        if (endpointRulesSpecUtils.isS3()) {
+            builder.addStatement("identityRequestBuilder.putProperty($T.SDK_CLIENT, "
+                                 + "executionAttributes.getAttribute($T.SDK_CLIENT))",
+                                 SdkIdentityProperty.class,
+                                 SdkInternalExecutionAttribute.class);
+        }
         builder.addStatement("$T identity", namedIdentityFuture());
         builder.addStatement("$T metric = getIdentityMetric(identityProvider)", durationSdkMetric());
         builder.beginControlFlow("if (metric == null)")
@@ -295,6 +304,43 @@ private MethodSpec generateGetIdentityMetric() {
         return builder.build();
     }
 
+    private MethodSpec putSelectedAuthSchemeMethodSpec() {
+        String attributeParamName = "attributes";
+        String selectedAuthSchemeParamName = "selectedAuthScheme";
+        MethodSpec.Builder builder = MethodSpec.methodBuilder("putSelectedAuthScheme")
+                                               .addModifiers(Modifier.PRIVATE)
+                                               .addTypeVariable(TypeVariableName.get("T", Identity.class))
+                                               .addParameter(ExecutionAttributes.class, attributeParamName)
+                                               .addParameter(ParameterSpec.builder(
+                                                   ParameterizedTypeName.get(ClassName.get(SelectedAuthScheme.class),
+                                                                             TypeVariableName.get("T")),
+                                                   selectedAuthSchemeParamName).build());
+        builder.addStatement("$T existingAuthScheme = $N.getAttribute($T.SELECTED_AUTH_SCHEME)",
+                             ParameterizedTypeName.get(ClassName.get(SelectedAuthScheme.class),
+                                                       WildcardTypeName.subtypeOf(Object.class)),
+                             attributeParamName,
+                             SdkInternalExecutionAttribute.class);
+
+        builder.beginControlFlow("if (existingAuthScheme != null)")
+               .addStatement("$T selectedOption = $N.authSchemeOption().toBuilder()",
+                             AuthSchemeOption.Builder.class, selectedAuthSchemeParamName)
+               .addStatement("existingAuthScheme.authSchemeOption().forEachIdentityProperty"
+                             + "(selectedOption::putIdentityPropertyIfAbsent)")
+               .addStatement("existingAuthScheme.authSchemeOption().forEachSignerProperty"
+                             + "(selectedOption::putSignerPropertyIfAbsent)")
+               .addStatement("$N = new $T<>($N.identity(), $N.signer(), selectedOption.build())",
+                             selectedAuthSchemeParamName,
+                             SelectedAuthScheme.class,
+                             selectedAuthSchemeParamName,
+                             selectedAuthSchemeParamName);
+        builder.endControlFlow();
+
+        builder.addStatement("$N.putAttribute($T.SELECTED_AUTH_SCHEME, $N)",
+                             attributeParamName, SdkInternalExecutionAttribute.class, selectedAuthSchemeParamName);
+
+        return builder.build();
+    }
+
     private void addLogDebugDiscardedOptions(MethodSpec.Builder builder) {
         builder.beginControlFlow("if (!discardedReasons.isEmpty())");
         {

codegen/src/main/java/software/amazon/awssdk/codegen/poet/auth/scheme/AuthSchemeSpecUtils.java

@@ -71,6 +71,10 @@ private String internalPackage() {
         return intermediateModel.getMetadata().getFullInternalAuthSchemePackageName();
     }
 
+    public String baseClientPackageName() {
+        return intermediateModel.getMetadata().getFullClientPackageName();
+    }
+
     public ClassName parametersInterfaceName() {
         return ClassName.get(basePackage(), intermediateModel.getMetadata().getServiceName() + "AuthSchemeParams");
     }

codegen/src/main/java/software/amazon/awssdk/codegen/poet/auth/scheme/EndpointBasedAuthSchemeProviderSpec.java

@@ -147,6 +147,9 @@ private void addAuthSchemeSwitch(MethodSpec.Builder spec) {
         spec.beginControlFlow("switch(name)");
         addAuthSchemeSwitchSigV4Case(spec);
         addAuthSchemeSwitchSigV4aCase(spec);
+        if (endpointRulesSpecUtils.useS3Express()) {
+            addAuthSchemeSwitchS3ExpressCase(spec);
+        }
         addAuthSchemeSwitchDefaultCase(spec);
         spec.endControlFlow();
     }
@@ -184,6 +187,26 @@ private void addAuthSchemeSwitchSigV4aCase(MethodSpec.Builder spec) {
         spec.addStatement("break");
     }
 
+    private void addAuthSchemeSwitchS3ExpressCase(MethodSpec.Builder spec) {
+        spec.addCode("case $S:", "sigv4-s3express");
+        ClassName s3ExpressEndpointAuthScheme = ClassName.get(
+            authSchemeSpecUtils.baseClientPackageName() + ".endpoints.authscheme",
+            "S3ExpressEndpointAuthScheme");
+        spec.addStatement("$T s3ExpressAuthScheme = $T.isInstanceOf($T.class, authScheme, $S, authScheme.getClass().getName())",
+                          s3ExpressEndpointAuthScheme, Validate.class, s3ExpressEndpointAuthScheme,
+                          "Expecting auth scheme of class S3ExpressAuthScheme, got instead object of class %s");
+
+        ClassName s3ExpressAuthScheme = ClassName.get(authSchemeSpecUtils.baseClientPackageName() + ".s3express",
+                                                      "S3ExpressAuthScheme");
+        spec.addCode("options.add($T.builder().schemeId($T.SCHEME_ID)", AuthSchemeOption.class, s3ExpressAuthScheme)
+            .addCode(".putSignerProperty($T.SERVICE_SIGNING_NAME, s3ExpressAuthScheme.signingName())", AwsV4HttpSigner.class)
+            .addCode(".putSignerProperty($T.REGION_NAME, s3ExpressAuthScheme.signingRegion())", AwsV4HttpSigner.class)
+            .addCode(".putSignerProperty($T.DOUBLE_URL_ENCODE, !s3ExpressAuthScheme.disableDoubleEncoding())",
+                     AwsV4HttpSigner.class)
+            .addCode(".build());");
+        spec.addStatement("break");
+    }
+
     private void addAuthSchemeSwitchDefaultCase(MethodSpec.Builder spec) {
         spec.addCode("default:");
         spec.addStatement("throw new $T($S + name)", IllegalArgumentException.class, "Unknown auth scheme: ");