fix(scripts): generate clients in batches (#3319)

Author: trivikr

Diff for: scripts/generate-clients/code-gen-dir.js

@@ -8,6 +8,7 @@ const CODE_GEN_SDK_ROOT = getCodeGenDirRoot("sdk-codegen");
 const CODE_GEN_PROTOCOL_TESTS_ROOT = getCodeGenDirRoot("protocol-test-codegen");
 
 const TEMP_CODE_GEN_INPUT_DIR = normalize(join(__dirname, ".aws-models"));
+const DEFAULT_CODE_GEN_INPUT_DIR = normalize(join(CODE_GEN_SDK_ROOT, "aws-models"));
 
 const getCodeGenOutputDir = (dir) =>
   normalize(join(__dirname, "..", "..", "codegen", dir, "build", "smithyprojections", dir));
@@ -22,5 +23,6 @@ module.exports = {
   CODE_GEN_SDK_OUTPUT_DIR,
   CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR,
   CODE_GEN_GENERIC_CLIENT_OUTPUT_DIR,
+  DEFAULT_CODE_GEN_INPUT_DIR,
   TEMP_CODE_GEN_INPUT_DIR,
 };

Diff for: scripts/generate-clients/code-gen.js

@@ -1,51 +1,28 @@
 // @ts-check
-const path = require("path");
+const { basename, join, relative } = require("path");
 const { emptyDirSync } = require("fs-extra");
-const { copyFileSync, readdirSync, lstatSync } = require("fs");
+const { copyFileSync } = require("fs");
 const { spawnProcess } = require("../utils/spawn-process");
 const { CODE_GEN_ROOT, CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR } = require("./code-gen-dir");
-const Glob = require("glob");
+const { getModelFilepaths } = require("./get-model-filepaths");
 
-const generateClients = async (models) => {
-  let designatedModels = false;
-  if (typeof models === "string") {
-    //`models` is a folder path
-    designatedModels = true;
+const generateClients = async (models, batchSize) => {
+  const filepaths = getModelFilepaths(models);
+  const options = [
+    ":sdk-codegen:clean",
+    ":sdk-codegen:build",
+    `-PmodelsDirProp=${relative(CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR)}`,
+  ];
+
+  while (filepaths.length > 0) {
     emptyDirSync(TEMP_CODE_GEN_INPUT_DIR);
-    console.log(`preparing models from ${models}...`);
-    for (const modelFileName of readdirSync(models)) {
-      const modelPath = path.join(models, modelFileName);
-      if (!lstatSync(modelPath).isFile()) continue;
-      console.log(`copying model ${modelFileName}...`);
-      copyFileSync(modelPath, path.join(TEMP_CODE_GEN_INPUT_DIR, modelFileName));
+    const filepathsToCopy = filepaths.splice(0, batchSize);
+    for (const filepath of filepathsToCopy) {
+      const filename = basename(filepath);
+      copyFileSync(filepath, join(TEMP_CODE_GEN_INPUT_DIR, filename));
     }
-  } else if (Array.isArray(models)) {
-    //`models` is a list of globs
-    designatedModels = true;
-    emptyDirSync(TEMP_CODE_GEN_INPUT_DIR);
-    models.forEach((pattern) => {
-      const files = Glob.sync(pattern, {
-        realpath: true,
-        absolute: true,
-      });
-      files.forEach((file) => {
-        if (!lstatSync(file).isFile()) return;
-        const name = path.basename(file);
-        console.log(`copying model ${name}...`);
-        copyFileSync(file, path.join(TEMP_CODE_GEN_INPUT_DIR, name));
-      });
-    });
-  } else {
-    console.log("no model supplied, generating all AWS clients");
-  }
-  const options = [":sdk-codegen:clean", ":sdk-codegen:build"];
-  if (designatedModels) {
-    options.push(`-PmodelsDirProp=${path.relative(CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR)}`);
+    await spawnProcess("./gradlew", options, { cwd: CODE_GEN_ROOT });
   }
-
-  await spawnProcess("./gradlew", options, {
-    cwd: CODE_GEN_ROOT,
-  });
 };
 
 const generateProtocolTests = async () => {

Diff for: scripts/generate-clients/get-model-filepaths.js

@@ -0,0 +1,27 @@
+const { readdirSync, lstatSync } = require("fs");
+const Glob = require("glob");
+const { join } = require("path");
+
+const getModelFilepaths = (models) => {
+  if (typeof models === "string") {
+    //`models` is a folder path
+    console.log(`preparing models from ${models}...`);
+    return readdirSync(models)
+      .map((modelFileName) => join(models, modelFileName))
+      .filter((modelFilepath) => lstatSync(modelFilepath).isFile());
+  } else if (Array.isArray(models)) {
+    //`models` is a list of globs
+    console.log(`preparing models from ${models}...`);
+    return models
+      .map((pattern) =>
+        Glob.sync(pattern, {
+          realpath: true,
+          absolute: true,
+        })
+      )
+      .flat()
+      .filter((modelFilepath) => lstatSync(modelFilepath).isFile());
+  }
+};
+
+module.exports = { getModelFilepaths };

Diff for: scripts/generate-clients/index.js

@@ -8,6 +8,7 @@ const {
   CODE_GEN_SDK_OUTPUT_DIR,
   CODE_GEN_GENERIC_CLIENT_OUTPUT_DIR,
   CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR,
+  DEFAULT_CODE_GEN_INPUT_DIR,
   TEMP_CODE_GEN_INPUT_DIR,
 } = require("./code-gen-dir");
 const { prettifyCode } = require("./code-prettify");
@@ -22,6 +23,7 @@ const {
   output: clientsDir,
   noPrivateClients,
   s: serverOnly,
+  batchSize,
 } = yargs
   .alias("m", "models")
   .string("m")
@@ -41,6 +43,10 @@ const {
   .boolean("s")
   .describe("s", "Generate server artifacts instead of client ones")
   .conflicts("s", ["m", "g", "n"])
+  .describe("b", "Batchsize for generating clients")
+  .number("b")
+  .alias("b", "batch-size")
+  .default("b", 50)
   .help().argv;
 
 (async () => {
@@ -58,7 +64,7 @@ const {
       return;
     }
 
-    await generateClients(models || globs);
+    await generateClients(models || globs || DEFAULT_CODE_GEN_INPUT_DIR, batchSize);
     if (!noPrivateClients) {
       await generateGenericClient();
       await generateProtocolTests();