Added support for Sync pagination

Author: varunnvs92

codegen-maven-plugin/src/main/java/software/amazon/awssdk/codegen/maven/plugin/GenerationMojo.java

@@ -35,6 +35,7 @@
 import software.amazon.awssdk.codegen.model.config.BasicCodeGenConfig;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
 import software.amazon.awssdk.codegen.model.intermediate.ServiceExamples;
+import software.amazon.awssdk.codegen.model.service.Paginators;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Waiters;
 import software.amazon.awssdk.codegen.utils.ModelLoaderUtils;
@@ -50,6 +51,7 @@ public class GenerationMojo extends AbstractMojo {
     private static final String CUSTOMIZATION_CONFIG_FILE = "customization.config";
     private static final String EXAMPLES_FILE = "examples-1.json";
     private static final String WAITERS_FILE = "waiters-2.json";
+    private static final String PAGINATORS_FILE = "paginators.json";
 
     @Parameter(property = "codeGenResources", defaultValue = "${basedir}/src/main/resources/codegen-resources/")
     private File codeGenResources;
@@ -75,6 +77,7 @@ public void execute() throws MojoExecutionException {
                                       .customizationConfig(loadCustomizationConfig(p))
                                       .serviceModel(loadServiceModel(p))
                                       .waitersModel(loadWaiterModel(p))
+                                      .paginatorsModel(loadPaginatorModel(p))
                                       .examplesModel(loadExamplesModel(p))
                                       .build());
             } catch (MojoExecutionException e) {
@@ -134,6 +137,10 @@ private Waiters loadWaiterModel(Path root) {
         return loadOptionalModel(Waiters.class, root.resolve(WAITERS_FILE)).orElse(Waiters.NONE);
     }
 
+    private Paginators loadPaginatorModel(Path root) {
+        return loadOptionalModel(Paginators.class, root.resolve(PAGINATORS_FILE)).orElse(Paginators.NONE);
+    }
+
     /**
      * Load required model from the project resources.
      */

codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java

@@ -77,6 +77,7 @@ public static Metadata constructMetadata(ServiceModel serviceModel,
                 .withModelPackageName(Utils.getModelPackageName(serviceName, customizationConfig))
                 .withTransformPackageName(Utils.getTransformPackageName(serviceName, customizationConfig))
                 .withRequestTransformPackageName(Utils.getRequestTransformPackageName(serviceName, customizationConfig))
+                .withPaginatorsPackageName(Utils.getPaginatorsPackageName(serviceName, customizationConfig))
                 .withSmokeTestsPackageName(Utils.getSmokeTestPackageName(serviceName, customizationConfig))
                 .withServiceAbbreviation(serviceMetadata.getServiceAbbreviation())
                 .withServiceFullName(serviceMetadata.getServiceFullName())

codegen/src/main/java/software/amazon/awssdk/codegen/AddOperations.java

@@ -30,6 +30,7 @@
 import software.amazon.awssdk.codegen.model.service.Member;
 import software.amazon.awssdk.codegen.model.service.Operation;
 import software.amazon.awssdk.codegen.model.service.Output;
+import software.amazon.awssdk.codegen.model.service.PaginatorDefinition;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Shape;
 import software.amazon.awssdk.codegen.naming.NamingStrategy;
@@ -41,10 +42,12 @@ final class AddOperations {
 
     private final ServiceModel serviceModel;
     private final NamingStrategy namingStrategy;
+    private final Map<String, PaginatorDefinition> paginators;
 
     AddOperations(IntermediateModelBuilder builder) {
         this.serviceModel = builder.getService();
         this.namingStrategy = builder.getNamingStrategy();
+        this.paginators = builder.getPaginators().getPaginators();
     }
 
     private static boolean isAuthenticated(Operation op) {
@@ -151,6 +154,7 @@ public Map<String, OperationModel> constructOperations() {
             operationModel.setDeprecated(op.isDeprecated());
             operationModel.setDocumentation(op.getDocumentation());
             operationModel.setIsAuthenticated(isAuthenticated(op));
+            operationModel.setPaginated(isPaginated(op));
 
             final Input input = op.getInput();
             if (input != null) {
@@ -219,4 +223,8 @@ private Integer getHttpStatusCode(ErrorMap error, Shape shape) {
     private Integer getHttpStatusCode(ErrorTrait errorTrait) {
         return errorTrait == null ? null : errorTrait.getHttpStatusCode();
     }
+
+    private boolean isPaginated(Operation op) {
+        return paginators.keySet().contains(op.getName()) && paginators.get(op.getName()).isValid();
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/C2jModels.java

@@ -18,6 +18,7 @@
 import software.amazon.awssdk.codegen.model.config.BasicCodeGenConfig;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
 import software.amazon.awssdk.codegen.model.intermediate.ServiceExamples;
+import software.amazon.awssdk.codegen.model.service.Paginators;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Waiters;
 
@@ -31,14 +32,17 @@ public class C2jModels {
     private final ServiceExamples examplesModel;
     private final BasicCodeGenConfig codeGenConfig;
     private final CustomizationConfig customizationConfig;
+    private final Paginators paginatorsModel;
 
     private C2jModels(ServiceModel serviceModel, Waiters waitersModel, ServiceExamples examplesModel,
-                      BasicCodeGenConfig codeGenConfig, CustomizationConfig customizationConfig) {
+                      BasicCodeGenConfig codeGenConfig, CustomizationConfig customizationConfig,
+                      Paginators paginatorsModel) {
         this.serviceModel = serviceModel;
         this.waitersModel = waitersModel;
         this.examplesModel = examplesModel;
         this.codeGenConfig = codeGenConfig;
         this.customizationConfig = customizationConfig;
+        this.paginatorsModel = paginatorsModel;
     }
 
     public static Builder builder() {
@@ -65,13 +69,18 @@ public CustomizationConfig customizationConfig() {
         return customizationConfig;
     }
 
+    public Paginators paginatorsModel() {
+        return paginatorsModel;
+    }
+
     public static class Builder {
 
         private ServiceModel serviceModel;
         private Waiters waitersModel;
         private ServiceExamples examplesModel;
         private BasicCodeGenConfig codeGenConfig;
         private CustomizationConfig customizationConfig;
+        private Paginators paginatorsModel;
 
         private Builder() {
         }
@@ -101,10 +110,16 @@ public Builder customizationConfig(CustomizationConfig customizationConfig) {
             return this;
         }
 
+        public Builder paginatorsModel(Paginators paginatorsModel) {
+            this.paginatorsModel = paginatorsModel;
+            return this;
+        }
+
         public C2jModels build() {
             final Waiters waiters = waitersModel != null ? waitersModel : Waiters.NONE;
+            final Paginators paginators = paginatorsModel != null ? paginatorsModel : Paginators.NONE;
             final ServiceExamples examples = examplesModel != null ? examplesModel : ServiceExamples.NONE;
-            return new C2jModels(serviceModel, waiters, examples, codeGenConfig, customizationConfig);
+            return new C2jModels(serviceModel, waiters, examples, codeGenConfig, customizationConfig, paginators);
         }
     }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/IntermediateModelBuilder.java

@@ -43,6 +43,7 @@
 import software.amazon.awssdk.codegen.model.intermediate.ShapeModel;
 import software.amazon.awssdk.codegen.model.service.AuthType;
 import software.amazon.awssdk.codegen.model.service.Operation;
+import software.amazon.awssdk.codegen.model.service.Paginators;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.naming.DefaultNamingStrategy;
 import software.amazon.awssdk.codegen.naming.NamingStrategy;
@@ -61,6 +62,7 @@ public class IntermediateModelBuilder {
     private final NamingStrategy namingStrategy;
     private final TypeUtils typeUtils;
     private final List<IntermediateModelShapeProcessor> shapeProcessors;
+    private final Paginators paginators;
 
     public IntermediateModelBuilder(C2jModels models) {
         this.customConfig = models.customizationConfig();
@@ -70,6 +72,7 @@ public IntermediateModelBuilder(C2jModels models) {
         this.namingStrategy = new DefaultNamingStrategy(service, customConfig);
         this.typeUtils = new TypeUtils(namingStrategy);
         this.shapeProcessors = createShapeProcessors();
+        this.paginators = models.paginatorsModel();
     }
 
 
@@ -116,7 +119,7 @@ public IntermediateModel build() throws IOException {
 
         IntermediateModel fullModel = new IntermediateModel(
             constructMetadata(service, codeGenConfig, customConfig), operations, shapes,
-            customConfig, examples, authorizers);
+            customConfig, examples, authorizers, paginators.getPaginators());
 
         customization.postprocess(fullModel);
 
@@ -131,7 +134,8 @@ public IntermediateModel build() throws IOException {
                                                                trimmedShapes,
                                                                fullModel.getCustomizationConfig(),
                                                                fullModel.getExamples(),
-                                                               fullModel.getCustomAuthorizers());
+                                                               fullModel.getCustomAuthorizers(),
+                                                               fullModel.getPaginators());
 
         linkMembersToShapes(trimmedModel);
         linkOperationsToInputOutputShapes(trimmedModel);
@@ -249,4 +253,7 @@ public TypeUtils getTypeUtils() {
         return typeUtils;
     }
 
+    public Paginators getPaginators() {
+        return paginators;
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/docs/OperationDocProvider.java

@@ -51,10 +51,12 @@ abstract class OperationDocProvider {
 
     protected final IntermediateModel model;
     protected final OperationModel opModel;
+    protected final PaginationDocs paginationDocs;
 
     OperationDocProvider(IntermediateModel model, OperationModel opModel) {
         this.model = model;
         this.opModel = opModel;
+        this.paginationDocs = new PaginationDocs(model, opModel);
     }
 
     /**

codegen/src/main/java/software/amazon/awssdk/codegen/docs/PaginationDocs.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.docs;
+
+import com.squareup.javapoet.ClassName;
+import com.squareup.javapoet.CodeBlock;
+import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
+import software.amazon.awssdk.codegen.model.intermediate.OperationModel;
+import software.amazon.awssdk.codegen.poet.PoetExtensions;
+import software.amazon.awssdk.codegen.utils.PaginatorUtils;
+
+public class PaginationDocs {
+
+    private final OperationModel operationModel;
+    private final PoetExtensions poetExtensions;
+
+    public PaginationDocs(IntermediateModel intermediateModel, OperationModel operationModel) {
+        this.operationModel = operationModel;
+        this.poetExtensions = new PoetExtensions(intermediateModel);
+    }
+
+    /**
+     * Constructs additional documentation on the client operation that is appended to the service documentation.
+     *
+     * TODO Add a link to our developer guide which will have more details and sample code. Write a blog post too.
+     */
+    public String getDocsForSyncOperation() {
+        return CodeBlock.builder()
+                        .add("<p>This is a variant of {@link #$L($T)} operation. "
+                             + "The return type is a custom iterable that can be used to iterate through all the pages. "
+                             + "SDK will internally handle making service calls for you.\n</p>",
+                             operationModel.getMethodName(), requestType())
+                        .add("<p>\nWhen this operation is called, a custom iterable is returned but no service calls are "
+                             + "made yet. So there is no guarantee that the request is valid. As you iterate "
+                             + "through the iterable, SDK will start lazily loading response pages by making service calls until "
+                             + "there are no pages left or your iteration stops. If there are errors in your request, you will "
+                             + "see the failures only after you start iterating through the iterable.\n</p>")
+                        .add(getSyncCodeSnippets())
+                        .build()
+                        .toString();
+    }
+
+    /**
+     * Constructs javadocs for the generated response classes in a paginated operation.
+     * @param clientInterface A java poet {@link ClassName} type of the sync client interface
+     */
+    public String getDocsForSyncResponseClass(ClassName clientInterface) {
+        return CodeBlock.builder()
+                        .add("<p>Represents the output for the {@link $T#$L($T)} operation which is a paginated operation."
+                             + " This class is an iterable of {@link $T} that can be used to iterate through all the "
+                             + "response pages of the operation.</p>",
+                             clientInterface, getSyncPaginatedMethodName(), requestType(), syncResponsePageType())
+                        .add("<p>When the operation is called, an instance of this class is returned.  At this point, "
+                             + "no service calls are made yet and so there is no guarantee that the request is valid. "
+                             + "As you iterate through the iterable, SDK will start lazily loading response pages by making "
+                             + "service calls until there are no pages left or your iteration stops. If there are errors in your "
+                             + "request, you will see the failures only after you start iterating through the iterable.</p>")
+                        .add(getSyncCodeSnippets())
+                        .build()
+                        .toString();
+    }
+
+    private String getSyncCodeSnippets() {
+        CodeBlock callOperationOnClient = CodeBlock.builder()
+                                           .addStatement("$T responses = client.$L(request)", syncPaginatedResponseType(),
+                                                         getSyncPaginatedMethodName())
+                                           .build();
+
+        return CodeBlock.builder()
+                        .add("\n\n<p>The following are few ways to iterate through the response pages:</p>")
+                        .add("1) Using a Stream")
+                        .add(buildCode(CodeBlock.builder()
+                                                .add(callOperationOnClient)
+                                                .addStatement("responses.stream().forEach(....)")
+                                                .build()))
+                        .add("\n\n2) Using For loop")
+                        .add(buildCode(CodeBlock.builder()
+                                                .add(callOperationOnClient)
+                                                .beginControlFlow("for ($T response : responses)", syncResponsePageType())
+                                                .addStatement(" // do something")
+                                                .endControlFlow()
+                                                .build()))
+                        .add("\n\n3) Use iterator directly")
+                        .add(buildCode(CodeBlock.builder()
+                                                .add(callOperationOnClient)
+                                                .addStatement("responses.iterator().forEachRemaining(....)")
+                                                .build()))
+                        .add(noteAboutSyncNonPaginatedMethod())
+                        .build()
+                        .toString();
+    }
+
+    private CodeBlock buildCode(CodeBlock codeSnippet) {
+        return CodeBlock.builder()
+                        .add("<pre>{@code\n")
+                        .add(codeSnippet)
+                        .add("}</pre>")
+                        .build();
+
+    }
+
+    /**
+     * @return Method name for the sync paginated operation
+     */
+    private String getSyncPaginatedMethodName() {
+        return PaginatorUtils.getSyncMethodName(operationModel.getMethodName());
+    }
+
+    /**
+     * @return A Poet {@link ClassName} for the sync operation request type.
+     *
+     * Example: For ListTables operation, it will be "ListTablesRequest" class.
+     */
+    private ClassName requestType() {
+        return poetExtensions.getModelClass(operationModel.getInput().getVariableType());
+    }
+
+    /**
+     * @return A Poet {@link ClassName} for the return type of sync non-paginated operation.
+     *
+     * Example: For ListTables operation, it will be "ListTablesResponse" class.
+     */
+    private ClassName syncResponsePageType() {
+        return poetExtensions.getModelClass(operationModel.getReturnType().getReturnType());
+    }
+
+    /**
+     * @return A Poet {@link ClassName} for the return type of sync paginated operation.
+     */
+    public ClassName syncPaginatedResponseType() {
+        return poetExtensions.getResponseClassForPaginatedSyncOperation(operationModel.getOperationName());
+    }
+
+    private CodeBlock noteAboutSyncNonPaginatedMethod() {
+        return CodeBlock.builder()
+                        .add("\n<p><b>Note: If you prefer to have control on service calls, use the {@link #$L($T)} operation."
+                             + "</b></p>", operationModel.getMethodName(), requestType())
+                        .build();
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/docs/SimpleMethodOverload.java

@@ -26,11 +26,21 @@ public enum SimpleMethodOverload {
      */
     NORMAL,
 
+    /**
+     * The standard paginated method overload that takes in a request object and returns a response object.
+     */
+    PAGINATED,
+
     /**
      * Simple method that takes no arguments and creates an empty request object that delegates to the {@link #NORMAL} overload.
      */
     NO_ARG,
 
+    /**
+     * Paginated simple method that takes no arguments and creates an empty request object.
+     */
+    NO_ARG_PAGINATED,
+
     /**
      * Simple method for streaming operations (input or output) that takes in the request object and a file to
      * upload from or download to.