chore: add yarn codegen script calling Gradle script (#545)

* chore: add generate-clients script generating clients from models path

* feat: clean the codegen input and output folder after generating

* fix: update lerna commands

use use '--include-dependencies' in replace of '--include-filtered-dependencies'

Author: AllanZhengYP

package.json

@@ -5,14 +5,13 @@
   "description": "AWS SDK for JavaScript from the future",
   "main": "index.js",
   "scripts": {
+    "generate-clients": "node ./scripts/generate-clients",
     "bootstrap": "yarn",
     "clean": "yarn clear-build-cache && yarn clear-build-info && lerna clean",
     "clear-build-cache": "rimraf ./packages/*/build/* ./clients/*/*/build/*",
     "clear-build-info": "rimraf ./packages/*/*.tsbuildinfo ./clients/*/*/*.tsbuildinfo",
-    "copy-models": "node ./scripts/copyModels.js",
-    "update-clients": "node ./packages/package-generator/build/cli.js import-all --matching './models/*/*/service-2.json'",
-    "build:crypto-dependencies": "lerna run --scope '@aws-sdk/types' --scope '@aws-sdk/util-utf8-browser' --scope '@aws-sdk/util-locate-window' --scope '@aws-sdk/hash-node' --include-filtered-dependencies pretest",
-    "build:smithy-client": "lerna run --scope '@aws-sdk/client-rds-data' --include-filtered-dependencies pretest",
+    "build:crypto-dependencies": "lerna run --scope '@aws-sdk/types' --scope '@aws-sdk/util-utf8-browser' --scope '@aws-sdk/util-locate-window' --scope '@aws-sdk/hash-node' --include-dependencies pretest",
+    "build:smithy-client": "lerna run --scope '@aws-sdk/client-rds-data' --include-dependencies pretest",
     "pretest": "yarn build:crypto-dependencies && yarn build:smithy-client",
     "test": "jest --coverage --passWithNoTests",
     "pretest-all": "lerna run pretest",
@@ -36,9 +35,11 @@
   "devDependencies": {
     "@commitlint/cli": "^8.1.0",
     "@commitlint/config-conventional": "^8.1.0",
+    "@types/fs-extra": "^8.0.1",
     "@types/jest": "^24.0.12",
     "codecov": "^3.4.0",
     "cucumber": "0.5.x",
+    "fs-extra": "^8.1.0",
     "generate-changelog": "^1.7.1",
     "husky": "^3.0.0",
     "jest": "^24.7.1",

scripts/copyModels.js

@@ -0,0 +1,65 @@
+const path = require("path");
+const { copyFileSync, emptyDirSync } = require("fs-extra");
+const { readdirSync, lstatSync } = require("fs");
+const { spawn } = require("child_process");
+
+const CODE_GEN_INPUT_DIR = path.normalize(
+  path.join(__dirname, "..", "..", "codegen", "sdk-codegen", "aws-models")
+);
+const CODE_GEN_ROOT = path.normalize(
+  path.join(__dirname, "..", "..", "codegen")
+);
+
+async function generateClients(models) {
+  console.info("models directory: ", models);
+  if (models === CODE_GEN_INPUT_DIR) {
+    // console.log("skipping copying models to codegen directory");
+    throw new Error(
+      `models directory cannot be the same as ${CODE_GEN_INPUT_DIR}`
+    );
+  } else {
+    console.log(`clearing code gen input folder...`);
+    emptyDirSync(CODE_GEN_INPUT_DIR);
+    console.log(`copying models from ${models} to ${CODE_GEN_INPUT_DIR}...`);
+    // copySync(models, CODE_GEN_INPUT_DIR, {
+    //   overwrite: true
+    // });
+    for (const modelFileName of readdirSync(models)) {
+      const modelPath = path.join(models, modelFileName);
+      if (!lstatSync(modelPath).isFile()) continue;
+      console.log(`copying model ${modelFileName}...`);
+      copyFileSync(modelPath, path.join(CODE_GEN_INPUT_DIR, modelFileName), {
+        overwrite: true
+      });
+    }
+  }
+  await spawnProcess(
+    "./gradlew",
+    [":sdk-codegen:clean", ":sdk-codegen:build"],
+    {
+      cwd: CODE_GEN_ROOT
+    }
+  );
+}
+
+const spawnProcess = (command, args = [], options = {}) =>
+  new Promise((resolve, reject) => {
+    try {
+      const ls = spawn(command, args, options);
+      ls.stdout.on("data", data => {
+        console.log(data.toString());
+      });
+      ls.stderr.on("data", data => {
+        console.error(`stderr: ${data.toString()}`);
+      });
+
+      ls.on("close", code => {
+        console.log(`child process exited with code ${code}`);
+        resolve();
+      });
+    } catch (e) {
+      reject(e);
+    }
+  });
+
+module.exports = { generateClients, CODE_GEN_INPUT_DIR };

scripts/generate-clients/code-gen.js

@@ -0,0 +1,89 @@
+const path = require("path");
+const { copySync, ensureDirSync } = require("fs-extra");
+const {
+  readdirSync,
+  lstatSync,
+  readFileSync,
+  existsSync,
+  writeFileSync
+} = require("fs");
+
+const CODE_GEN_OUTPUT_DIR = path.normalize(
+  path.join(
+    __dirname,
+    "..",
+    "..",
+    "codegen",
+    "sdk-codegen",
+    "build",
+    "smithyprojections",
+    "sdk-codegen"
+  )
+);
+
+const unOverridables = [
+  "package.json",
+  "tsconfig.es.json",
+  "tsconfig.json",
+  "tsconfig.test.json"
+];
+
+async function copyToClients(clientsDir) {
+  for (const modelName of readdirSync(CODE_GEN_OUTPUT_DIR)) {
+    if (modelName === "source") continue;
+    const artifactPath = path.join(
+      CODE_GEN_OUTPUT_DIR,
+      modelName,
+      "typescript-codegen"
+    );
+    const packageManifestPath = path.join(artifactPath, "package.json");
+    if (!existsSync(packageManifestPath)) {
+      console.error(`${modelName} generates empty client, skip.`);
+      continue;
+    }
+    const packageManifest = JSON.parse(
+      readFileSync(packageManifestPath).toString()
+    );
+    const packageName = packageManifest.name.replace("@aws-sdk/", "");
+    console.log(`copying ${packageName} from ${artifactPath} to ${clientsDir}`);
+    const destPath = path.join(clientsDir, packageName);
+    for (const packageSub of readdirSync(artifactPath)) {
+      const packageSubPath = path.join(artifactPath, packageSub);
+      const destSubPath = path.join(destPath, packageSub);
+      if (unOverridables.indexOf(packageSub) >= 0) {
+        if (!existsSync(destSubPath))
+          copySync(packageSubPath, destSubPath, { overwrite: true });
+        else if (packageSub === "package.json") {
+          /**
+           * Copy package.json content in detail.
+           * Basically merge the generated package.json and dest package.json
+           * but prefer the values from dest when they contain the same key
+           * */
+          const destManifest = JSON.parse(readFileSync(destSubPath).toString());
+          const updatedManifest = {
+            ...packageManifest,
+            ...destManifest,
+            scripts: {
+              ...packageManifest.scripts,
+              ...destManifest.scripts
+            },
+            dependencies: {
+              ...packageManifest.dependencies,
+              ...destManifest.dependencies
+            },
+            devDependencies: {
+              ...packageManifest.devDependencies,
+              ...destManifest.devDependencies
+            }
+          };
+          writeFileSync(destSubPath, JSON.stringify(updatedManifest, null, 2));
+        }
+      } else {
+        if (lstatSync(packageSubPath).isDirectory()) ensureDirSync(destSubPath);
+        copySync(packageSubPath, destSubPath, { overwrite: true });
+      }
+    }
+  }
+}
+
+module.exports = { copyToClients, CODE_GEN_OUTPUT_DIR };

scripts/generate-clients/copy-to-clients.js

@@ -0,0 +1,30 @@
+const yargs = require("yargs");
+const path = require("path");
+const { emptyDirSync } = require("fs-extra");
+const { generateClients, CODE_GEN_INPUT_DIR } = require("./code-gen");
+const { copyToClients, CODE_GEN_OUTPUT_DIR } = require("./copy-to-clients");
+
+const CLIENTS_DIR = path.normalize(path.join(__dirname, "..", "..", "clients"));
+
+const { models, output: clientsDir } = yargs
+  .alias("m", "models")
+  .string("m")
+  .describe("m", "the directory of models")
+  .required("m")
+  .alias("o", "output")
+  .string("o")
+  .describe("o", "the output directory for built clients")
+  .default("o", CLIENTS_DIR)
+  .help().argv;
+
+(async () => {
+  try {
+    await generateClients(models);
+    await copyToClients(clientsDir);
+    emptyDirSync(CODE_GEN_INPUT_DIR);
+    emptyDirSync(CODE_GEN_OUTPUT_DIR);
+  } catch (e) {
+    console.log(e);
+    process.exit(1);
+  }
+})();