Add support for Endpoints 2.0 (#3502)

This release updates internal logic for constructing API endpoints.

We have added rule-based endpoint providers. Additionally there are new
interfaces for endpoint providers and parameters at the service client level.

The SDK provides service-specific `EndpointProvider` interfaces (e.g.
`S3EndpointProvider`), that have a single method `resolveEndpoint`.
`resolveEndpoint` accepts a set of parameters (also service specific) and
computes an endpoint based on those parameters. All clients have a default
implementation, but users are free to implement these to customize their
endpoint logic.

Author: dagnir

.brazil.json

@@ -42,6 +42,9 @@
             "artifactType": "JAR",
             "includes": ["target/aws-sdk-java-third-party-jackson-dataformat-cbor-*.jar"]
         },
+        "ruleset-testing-core": {
+            "packageName": "AwsJavaSdk-Test-RuleSetTestingCore"
+        },
 
         "archetype-app-quickstart": { "skipImport": true },
         "archetype-lambda": { "skipImport": true },
@@ -113,6 +116,10 @@
         "software.amazon.awssdk.crt:aws-crt": { "packageName": "Aws-crt-java", "packageVersion": "1.0.x" },
         "software.amazon.eventstream:eventstream": { "packageName": "AwsFlowJava", "packageVersion": "1.0" },
         "software.amazon.ion:ion-java": { "packageName": "Maven-software-amazon-ion_ion-java", "packageVersion": "1.x" },
-        "software.amazon:flow": { "packageName": "AwsFlowJava", "packageVersion": "1.0" }
+        "software.amazon:flow": { "packageName": "AwsFlowJava", "packageVersion": "1.0" },
+        "org.junit.jupiter:junit-jupiter": { "packageName": "JUnit5", "packageVersion": "5.x" },
+        "org.mockito:mockito-core": { "packageName": "Mockito", "packageVersion": "3.x" },
+        "io.reactivex.rxjava3:rxjava": { "packageName": "RxJava", ",packageVersion": "3.x" },
+        "org.assertj:assertj-core": { "packageName": "AssertJ", ",packageVersion": "3.22.x" }
     }
 }

.changes/next-release/feature-AWSSDKforJavav2-65cafca.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "This release updates internal logic for constructing API endpoints. \n\nWe have added rule-based endpoint providers. Additionally there are new interfaces for endpoint providers and parameters at the service client level.\n\nThe SDK provides service-specific `EndpointProvider` interfaces (e.g. `S3EndpointProvider`), that have a single method `resolveEndpoint`. `resolveEndpoint` accepts a set of parameters (also service specific) and computes an endpoint based on those parameters. All clients have a default implementation, but users are free to implement these to customize their endpoint logic.\n\nIt is important to know that `EndpointProvider`s and the endpoint override on the client are not mutually exclusive. In all existing cases, the endpoint override is passed as a parameter to the the provider and the provider *may* modify it. For example, the S3 provider may add the bucket name as a prefix to the endpoint override for virtual bucket addressing.\n\nThere may be some behavior changes related to endpoint resolution as as the the endpoint resolution is now standardised across SDKs so this release also bumps the minor version of the SDK to `2.18`.\n\nNotably, there are some changes in S3: when including and endpoint override, the SDK will use virtual bucket addressing as long as the bucket is a valid DNS label. This is different from the previous behavior where this only happens if the endpoint override begins with 's3'.\n\nIn many situations, the exception thrown for validations like including an endpoint override and enabling DualStack has changed to `SdkClientExcpetion` and error messages have changed as well."
+}

buildspecs/release-javadoc.yml

@@ -18,7 +18,7 @@ phases:
     commands:
     - python ./scripts/doc_crosslinks/generate_cross_link_data.py --apiDefinitionsBasePath ./services/ --apiDefinitionsRelativeFilePath src/main/resources/codegen-resources/service-2.json  --templateFilePath ./scripts/doc_crosslinks/crosslink_redirect.html --outputFilePath ./scripts/crosslink_redirect.html
     - mvn install -P quick -T1C
-    - mvn clean install javadoc:aggregate -B -Ppublic-javadoc -Dcheckstyle.skip -Dspotbugs.skip -DskipTests -Ddoclint=none -pl '!:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:s3-benchmarks,!:module-path-tests,!:test-utils,!:http-client-tests,!:tests-coverage-reporting,!:sdk-native-image-test'
+    - mvn clean install javadoc:aggregate -B -Ppublic-javadoc -Dcheckstyle.skip -Dspotbugs.skip -DskipTests -Ddoclint=none -pl '!:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:s3-benchmarks,!:module-path-tests,!:test-utils,!:http-client-tests,!:tests-coverage-reporting,!:sdk-native-image-test,!:ruleset-testing-core'
     - RELEASE_VERSION=`mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec`
     -
     - aws s3 sync target/site/apidocs/ $DOC_PATH/$RELEASE_VERSION/

buildspecs/release-to-maven.yml

@@ -40,7 +40,7 @@ phases:
             awk 'BEGIN { var=ENVIRON["SDK_SIGNING_GPG_KEYNAME"] } { gsub("\\$SDK_SIGNING_GPG_KEYNAME", var, $0); print }' > \
             $SETTINGS_XML
 
-          mvn clean deploy -B -s $SETTINGS_XML -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-tests,!:s3-benchmarks,!:region-testing -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30
+          mvn clean deploy -B -s $SETTINGS_XML -Ppublishing -DperformRelease -Dspotbugs.skip -DskipTests -Dcheckstyle.skip -Djapicmp.skip -Ddoclint=none -pl !:protocol-tests,!:protocol-tests-core,!:codegen-generated-classes-test,!:sdk-benchmarks,!:module-path-tests,!:tests-coverage-reporting,!:stability-tests,!:sdk-native-image-test,!:auth-tests,!:s3-benchmarks,!:region-testing,!:ruleset-testing-core -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30
         else
           echo "This version was already released."
         fi

codegen-maven-plugin/pom.xml

@@ -57,6 +57,16 @@
             <groupId>software.amazon.awssdk</groupId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <scope>compile</scope>
+        </dependency>
         <dependency>
             <artifactId>maven-plugin-api</artifactId>
             <groupId>org.apache.maven</groupId>

codegen-maven-plugin/src/main/java/software/amazon/awssdk/codegen/maven/plugin/GenerationMojo.java

@@ -32,6 +32,8 @@
 import software.amazon.awssdk.codegen.CodeGenerator;
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
+import software.amazon.awssdk.codegen.model.rules.endpoints.EndpointTestSuiteModel;
+import software.amazon.awssdk.codegen.model.service.EndpointRuleSetModel;
 import software.amazon.awssdk.codegen.model.service.Paginators;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Waiters;
@@ -47,6 +49,9 @@ public class GenerationMojo extends AbstractMojo {
     private static final String CUSTOMIZATION_CONFIG_FILE = "customization.config";
     private static final String WAITERS_FILE = "waiters-2.json";
     private static final String PAGINATORS_FILE = "paginators-1.json";
+    private static final String ENDPOINT_RULE_SET_FILE = "endpoint-rule-set.json";
+    private static final String ENDPOINT_TESTS_FILE = "endpoint-tests.json";
+
 
     @Parameter(property = "codeGenResources", defaultValue = "${basedir}/src/main/resources/codegen-resources/")
     private File codeGenResources;
@@ -61,11 +66,13 @@ public class GenerationMojo extends AbstractMojo {
     private MavenProject project;
 
     private Path sourcesDirectory;
+    private Path resourcesDirectory;
     private Path testsDirectory;
 
     @Override
     public void execute() throws MojoExecutionException {
         this.sourcesDirectory = Paths.get(outputDirectory).resolve("generated-sources").resolve("sdk");
+        this.resourcesDirectory = Paths.get(outputDirectory).resolve("generated-resources").resolve("sdk-resources");
         this.testsDirectory = Paths.get(outputDirectory).resolve("generated-test-sources").resolve("sdk-tests");
 
         findModelRoots().forEach(p -> {
@@ -76,6 +83,8 @@ public void execute() throws MojoExecutionException {
                                   .serviceModel(loadServiceModel(modelRootPath))
                                   .waitersModel(loadWaiterModel(modelRootPath))
                                   .paginatorsModel(loadPaginatorModel(modelRootPath))
+                                  .endpointRuleSetModel(loadEndpointRuleSetModel(modelRootPath))
+                                  .endpointTestSuiteModel(loadEndpointTestSuiteModel(modelRootPath))
                                   .build());
         });
         project.addCompileSourceRoot(sourcesDirectory.toFile().getAbsolutePath());
@@ -105,6 +114,7 @@ private void generateCode(C2jModels models) {
         CodeGenerator.builder()
                      .models(models)
                      .sourcesDirectory(sourcesDirectory.toFile().getAbsolutePath())
+                     .resourcesDirectory(resourcesDirectory.toFile().getAbsolutePath())
                      .testsDirectory(testsDirectory.toFile().getAbsolutePath())
                      .intermediateModelFileNamePrefix(intermediateModelFileNamePrefix(models))
                      .build()
@@ -134,6 +144,14 @@ private Paginators loadPaginatorModel(Path root) {
         return loadOptionalModel(Paginators.class, root.resolve(PAGINATORS_FILE)).orElse(Paginators.none());
     }
 
+    private EndpointRuleSetModel loadEndpointRuleSetModel(Path root) {
+        return loadRequiredModel(EndpointRuleSetModel.class, root.resolve(ENDPOINT_RULE_SET_FILE));
+    }
+
+    private EndpointTestSuiteModel loadEndpointTestSuiteModel(Path root) {
+        return loadRequiredModel(EndpointTestSuiteModel.class, root.resolve(ENDPOINT_TESTS_FILE));
+    }
+
     /**
      * Load required model from the project resources.
      */

codegen/pom.xml

@@ -62,6 +62,11 @@
             <artifactId>metrics-spi</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>endpoints-spi</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>regions</artifactId>
@@ -117,6 +122,11 @@
             <artifactId>protocol-core</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>ruleset-testing-core</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
 
         <dependency>
             <artifactId>org.eclipse.jdt.core</artifactId>
@@ -158,7 +168,7 @@
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>
-            <scope>test</scope>
+            <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>org.junit.vintage</groupId>
@@ -178,7 +188,7 @@
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
-            <scope>test</scope>
+            <scope>compile</scope>
         </dependency>
     </dependencies>
 </project>

codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java

@@ -58,6 +58,7 @@ public static Metadata constructMetadata(ServiceModel serviceModel,
                 .withRequestTransformPackageName(namingStrategy.getRequestTransformPackageName(serviceName))
                 .withPaginatorsPackageName(namingStrategy.getPaginatorsPackageName(serviceName))
                 .withWaitersPackageName(namingStrategy.getWaitersPackageName(serviceName))
+                .withEndpointRulesPackageName(namingStrategy.getEndpointRulesPackageName(serviceName))
                 .withServiceAbbreviation(serviceMetadata.getServiceAbbreviation())
                 .withServiceFullName(serviceMetadata.getServiceFullName())
                 .withServiceName(serviceName)

codegen/src/main/java/software/amazon/awssdk/codegen/AddOperations.java

@@ -165,6 +165,7 @@ public Map<String, OperationModel> constructOperations() {
             operationModel.setEndpointTrait(op.getEndpoint());
             operationModel.setHttpChecksumRequired(op.isHttpChecksumRequired());
             operationModel.setHttpChecksum(op.getHttpChecksum());
+            operationModel.setStaticContextParams(op.getStaticContextParams());
 
             Input input = op.getInput();
             if (input != null) {

codegen/src/main/java/software/amazon/awssdk/codegen/AddShapes.java

@@ -191,6 +191,7 @@ private MemberModel generateMemberModel(String c2jMemberName, Member c2jMemberDe
         memberModel.setEndpointDiscoveryId(c2jMemberDefinition.isEndpointdiscoveryid());
         memberModel.setXmlAttribute(c2jMemberDefinition.isXmlAttribute());
         memberModel.setUnionEnumTypeName(namingStrategy.getUnionEnumTypeName(memberModel));
+        memberModel.setContextParam(c2jMemberDefinition.getContextParam());
 
 
         // Pass the xmlNameSpace from the member reference

codegen/src/main/java/software/amazon/awssdk/codegen/C2jModels.java

@@ -16,6 +16,8 @@
 package software.amazon.awssdk.codegen;
 
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
+import software.amazon.awssdk.codegen.model.rules.endpoints.EndpointTestSuiteModel;
+import software.amazon.awssdk.codegen.model.service.EndpointRuleSetModel;
 import software.amazon.awssdk.codegen.model.service.Paginators;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Waiters;
@@ -28,15 +30,21 @@ public class C2jModels {
 
     private final ServiceModel serviceModel;
     private final Waiters waitersModel;
+    private final EndpointRuleSetModel endpointRuleSetModel;
+    private final EndpointTestSuiteModel endpointTestSuiteModel;
     private final CustomizationConfig customizationConfig;
     private final Paginators paginatorsModel;
 
     private C2jModels(ServiceModel serviceModel,
                       Waiters waitersModel,
+                      EndpointRuleSetModel endpointRuleSetModel,
+                      EndpointTestSuiteModel endpointTestSuiteModel,
                       CustomizationConfig customizationConfig,
                       Paginators paginatorsModel) {
         this.serviceModel = serviceModel;
         this.waitersModel = waitersModel;
+        this.endpointRuleSetModel = endpointRuleSetModel;
+        this.endpointTestSuiteModel = endpointTestSuiteModel;
         this.customizationConfig = customizationConfig;
         this.paginatorsModel = paginatorsModel;
     }
@@ -61,10 +69,20 @@ public Paginators paginatorsModel() {
         return paginatorsModel;
     }
 
+    public EndpointRuleSetModel endpointRuleSetModel() {
+        return endpointRuleSetModel;
+    }
+
+    public EndpointTestSuiteModel endpointTestSuiteModel() {
+        return endpointTestSuiteModel;
+    }
+
     public static class Builder implements SdkBuilder<Builder, C2jModels> {
 
         private ServiceModel serviceModel;
         private Waiters waitersModel;
+        private EndpointRuleSetModel endpointRuleSetModel;
+        private EndpointTestSuiteModel endpointTestSuiteModel;
         private CustomizationConfig customizationConfig;
         private Paginators paginatorsModel;
 
@@ -91,11 +109,22 @@ public Builder paginatorsModel(Paginators paginatorsModel) {
             return this;
         }
 
+        public Builder endpointRuleSetModel(EndpointRuleSetModel endpointRuleSetModel) {
+            this.endpointRuleSetModel = endpointRuleSetModel;
+            return this;
+        }
+
+        public Builder endpointTestSuiteModel(EndpointTestSuiteModel endpointTestSuiteModel) {
+            this.endpointTestSuiteModel = endpointTestSuiteModel;
+            return this;
+        }
+
         @Override
         public C2jModels build() {
             Waiters waiters = waitersModel != null ? waitersModel : Waiters.none();
             Paginators paginators = paginatorsModel != null ? paginatorsModel : Paginators.none();
-            return new C2jModels(serviceModel, waiters, customizationConfig, paginators);
+            return new C2jModels(serviceModel, waiters, endpointRuleSetModel, endpointTestSuiteModel, customizationConfig,
+                                 paginators);
         }
     }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/CodeGenerator.java

@@ -33,6 +33,7 @@ public class CodeGenerator {
 
     private final C2jModels models;
     private final String sourcesDirectory;
+    private final String resourcesDirectory;
     private final String testsDirectory;
 
     /**
@@ -51,6 +52,7 @@ public CodeGenerator(Builder builder) {
         this.models = builder.models;
         this.sourcesDirectory = builder.sourcesDirectory;
         this.testsDirectory = builder.testsDirectory;
+        this.resourcesDirectory = builder.resourcesDirectory;
         this.fileNamePrefix = builder.fileNamePrefix;
     }
 
@@ -124,7 +126,10 @@ private void emitCode(IntermediateModel intermediateModel) {
     }
 
     private GeneratorTask createGeneratorTasks(IntermediateModel intermediateModel) {
-        return new AwsGeneratorTasks(GeneratorTaskParams.create(intermediateModel, sourcesDirectory, testsDirectory));
+        return new AwsGeneratorTasks(GeneratorTaskParams.create(intermediateModel,
+                                                                sourcesDirectory,
+                                                                testsDirectory,
+                                                                resourcesDirectory));
 
     }
 
@@ -135,6 +140,7 @@ public static final class Builder {
 
         private C2jModels models;
         private String sourcesDirectory;
+        private String resourcesDirectory;
         private String testsDirectory;
         private String fileNamePrefix;
 
@@ -151,6 +157,11 @@ public Builder sourcesDirectory(String sourcesDirectory) {
             return this;
         }
 
+        public Builder resourcesDirectory(String resourcesDirectory) {
+            this.resourcesDirectory = resourcesDirectory;
+            return this;
+        }
+
         public Builder testsDirectory(String smokeTestsDirectory) {
             this.testsDirectory = smokeTestsDirectory;
             return this;