chore: add parameter to gradle task and update code-gen script (#976)

* chore: smithy gradle can generated from designated folder

* chore: generate-clients scripts supports globs

Author: AllanZhengYP

codegen/sdk-codegen/build.gradle.kts

@@ -41,15 +41,17 @@ tasks.create<SmithyBuild>("buildSdk") {
 tasks.register("generate-smithy-build") {
     doLast {
         val projectionsBuilder = Node.objectNodeBuilder()
+        val modelsDirProp: String by project
+        val models = File(modelsDirProp)
 
-        fileTree("aws-models").filter { it.isFile }.files.forEach { file ->
+        fileTree(models).filter { it.isFile }.files.forEach { file ->
             val (sdkId, version, remaining) = file.name.split(".")
             var manifestOverwrites = Node.parse(
                     File("smithy-aws-typescript-codegen/src/main/resources/software/amazon/smithy/aws/typescript/codegen/package.json.template")
                             .readText()
             ).expectObjectNode()
             val projectionContents = Node.objectNodeBuilder()
-                    .withMember("imports", Node.fromStrings("aws-models/" + file.name))
+                    .withMember("imports", Node.fromStrings("${models.getAbsolutePath()}${File.separator}${file.name}"))
                     .withMember("plugins", Node.objectNode()
                             .withMember("typescript-codegen", Node.objectNodeBuilder()
                                     .withMember("package", "@aws-sdk/client-" + sdkId.toLowerCase())

codegen/sdk-codegen/gradle.properties

@@ -0,0 +1 @@
+modelsDirProp=aws-models

package.json

@@ -28,7 +28,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "glob": "^7.1.2",
+    "glob": "^7.1.6",
     "rimraf": "^2.6.2",
     "yargs": "^15.1.0"
   },
@@ -69,4 +69,4 @@
   "lint-staged": {
     "**/*.{ts,js,md,json}": "prettier --write"
   }
-}
+}

scripts/generate-clients/README.md

@@ -0,0 +1,29 @@
+# How to generate clients from Smithy model
+
+## Prerequisite
+
+This script requires Node.js >= 12
+
+## Options:
+
+```
+  -m, --models  The directory of models.                                [string]
+  -g, --globs   A list of smithy model globs                             [array]
+  -o, --output  The output directory for built clients
+           [string] [default: "../../clients"]
+  --help        Show help                                              [boolean]
+```
+
+## Examples:
+
+- Generate clients from folder with models:
+
+  `yarn generate-clients -m path/to/model`
+
+- Generate clients from a glob (e.g. generating S3, S3Control and DynamoDB clients):
+
+  `yarn generate-clients -g path/to/models/s3*.json path/to/another/dynamodb*.json`
+
+- Generate all known AWS clients:
+
+  `yarn generate-clients`

scripts/generate-clients/code-gen-dir.js

@@ -2,9 +2,7 @@ const { join, normalize } = require("path");
 
 const CODE_GEN_ROOT = normalize(join(__dirname, "..", "..", "codegen"));
 
-const CODE_GEN_INPUT_DIR = normalize(
-  join(__dirname, "..", "..", "codegen", "sdk-codegen", "aws-models")
-);
+const TEMP_CODE_GEN_INPUT_DIR = normalize(join(__dirname, ".aws-models"));
 
 const CODE_GEN_OUTPUT_DIR = normalize(
   join(
@@ -21,6 +19,6 @@ const CODE_GEN_OUTPUT_DIR = normalize(
 
 module.exports = {
   CODE_GEN_ROOT,
-  CODE_GEN_INPUT_DIR,
-  CODE_GEN_OUTPUT_DIR
+  CODE_GEN_OUTPUT_DIR,
+  TEMP_CODE_GEN_INPUT_DIR
 };

scripts/generate-clients/code-gen.js

@@ -2,36 +2,56 @@ const path = require("path");
 const { copyFileSync, emptyDirSync } = require("fs-extra");
 const { readdirSync, lstatSync } = require("fs");
 const { spawnProcess } = require("./spawn-process");
-const { CODE_GEN_ROOT, CODE_GEN_INPUT_DIR } = require("./code-gen-dir");
+const { CODE_GEN_ROOT, TEMP_CODE_GEN_INPUT_DIR } = require("./code-gen-dir");
+const Glob = require("glob");
 
 async function generateClients(models) {
-  console.info("models directory: ", models);
-  if (models === CODE_GEN_INPUT_DIR) {
-    // This script will clean the CODE_GEN_INPUT_DIR in execution.
-    // throw to avoid input model being removed
-    throw new Error(
-      `models directory cannot be the same as ${CODE_GEN_INPUT_DIR}`
-    );
-  } else {
-    console.log(`clearing code gen input folder...`);
-    emptyDirSync(CODE_GEN_INPUT_DIR);
-    console.log(`copying models from ${models} to ${CODE_GEN_INPUT_DIR}...`);
+  let designatedModels = false;
+  if (typeof models === "string") {
+    //`models` is a folder path
+    designatedModels = true;
+    emptyDirSync(TEMP_CODE_GEN_INPUT_DIR);
+    console.log(`preparing models from ${models}...`);
     for (const modelFileName of readdirSync(models)) {
       const modelPath = path.join(models, modelFileName);
       if (!lstatSync(modelPath).isFile()) continue;
       console.log(`copying model ${modelFileName}...`);
-      copyFileSync(modelPath, path.join(CODE_GEN_INPUT_DIR, modelFileName), {
-        overwrite: true
-      });
+      copyFileSync(
+        modelPath,
+        path.join(TEMP_CODE_GEN_INPUT_DIR, modelFileName),
+        {
+          overwrite: true
+        }
+      );
     }
+  } else if (Array.isArray(models)) {
+    //`models` is a list of globs
+    designatedModels = true;
+    emptyDirSync(TEMP_CODE_GEN_INPUT_DIR);
+    models.forEach(pattern => {
+      const files = Glob.sync(pattern, {
+        realpath: true,
+        absolute: true
+      });
+      files.forEach(file => {
+        if (!lstatSync(file).isFile()) return;
+        const name = path.basename(file);
+        console.log(`copying model ${name}...`);
+        copyFileSync(file, path.join(TEMP_CODE_GEN_INPUT_DIR, name), {
+          overwrite: true
+        });
+      });
+    });
+  } else {
+    console.log("no model supplied, generating all AWS clients");
   }
-  await spawnProcess(
-    "./gradlew",
-    [":sdk-codegen:clean", ":sdk-codegen:build"],
-    {
-      cwd: CODE_GEN_ROOT
-    }
-  );
+  const options = [":sdk-codegen:clean", ":sdk-codegen:build"];
+  if (designatedModels) {
+    options.push(`-PmodelsDirProp=${TEMP_CODE_GEN_INPUT_DIR}`);
+  }
+  await spawnProcess("./gradlew", options, {
+    cwd: CODE_GEN_ROOT
+  });
 }
 
 module.exports = {

scripts/generate-clients/index.js

@@ -1,31 +1,38 @@
 const yargs = require("yargs");
 const path = require("path");
-const { emptyDirSync } = require("fs-extra");
+const { emptyDirSync, rmdirSync } = require("fs-extra");
 const { generateClients } = require("./code-gen");
 const { copyToClients } = require("./copy-to-clients");
-const { CODE_GEN_INPUT_DIR, CODE_GEN_OUTPUT_DIR } = require("./code-gen-dir");
+const {
+  CODE_GEN_OUTPUT_DIR,
+  TEMP_CODE_GEN_INPUT_DIR
+} = require("./code-gen-dir");
 const { prettifyCode } = require("./code-prettify");
 
 const CLIENTS_DIR = path.normalize(path.join(__dirname, "..", "..", "clients"));
 
-const { models, output: clientsDir } = yargs
+const { models, globs, output: clientsDir } = yargs
   .alias("m", "models")
   .string("m")
-  .describe("m", "the directory of models")
-  .required("m")
+  .describe("m", "The directory of models.")
+  .alias("g", "globs")
+  .array("g")
+  .describe("g", "A list of smithy model globs")
+  .conflicts("models", "globs") //either models(path) or globs is accepted
   .alias("o", "output")
   .string("o")
-  .describe("o", "the output directory for built clients")
+  .describe("o", "The output directory for built clients")
   .default("o", CLIENTS_DIR)
   .help().argv;
 
 (async () => {
   try {
-    await generateClients(models);
+    await generateClients(models || globs);
     await prettifyCode();
     await copyToClients(clientsDir);
-    emptyDirSync(CODE_GEN_INPUT_DIR);
     emptyDirSync(CODE_GEN_OUTPUT_DIR);
+    emptyDirSync(TEMP_CODE_GEN_INPUT_DIR);
+    rmdirSync(TEMP_CODE_GEN_INPUT_DIR);
   } catch (e) {
     console.log(e);
     process.exit(1);