fix(scripts): run downlevel-dts script in parallel (#2837)

Author: trivikr

package.json

@@ -6,7 +6,7 @@
   "main": "index.js",
   "scripts": {
     "copy-models": "node ./scripts/copy-models",
-    "downlevel-dts": "node ./scripts/downlevel-dts",
+    "downlevel-dts": "node --es-module-specifier-resolution=node ./scripts/downlevel-dts",
     "generate-clients": "node ./scripts/generate-clients",
     "bootstrap": "yarn",
     "clean": "yarn clear-build-cache && yarn clear-build-info && lerna clean",
@@ -53,6 +53,7 @@
     "@types/jest": "^26.0.4",
     "@typescript-eslint/eslint-plugin": "4.30.0",
     "@typescript-eslint/parser": "4.30.0",
+    "async": "3.2.1",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "codecov": "^3.4.0",

scripts/downlevel-dts/README.md

@@ -10,7 +10,7 @@
 Runs downlevel-dts npm script (if present) in each workspace of monorepo, and
 strips comments from *.d.ts files.
 
-Usage: index.js
+Usage: index.mjs
 
 Options:
   --version   Show version number                                      [boolean]

scripts/downlevel-dts/downlevelWorkspace.mjs

@@ -0,0 +1,53 @@
+// @ts-check
+import { exec } from "child_process";
+import { access, readFile, writeFile } from "fs/promises";
+import { join } from "path";
+import stripComments from "strip-comments";
+import { promisify } from "util";
+
+import { getAllFiles } from "./getAllFiles.mjs";
+import { getDeclarationDirname } from "./getDeclarationDirname.mjs";
+import { getDownlevelDirname } from "./getDownlevelDirname.mjs";
+
+const execPromise = promisify(exec);
+
+export const downlevelWorkspace = async (workspacesDir, workspaceName) => {
+  const workspaceDir = join(workspacesDir, workspaceName);
+  const downlevelDirname = await getDownlevelDirname(workspaceDir);
+  const declarationDirname = await getDeclarationDirname(workspaceDir);
+
+  const declarationDir = join(workspaceDir, declarationDirname);
+  try {
+    await access(declarationDir);
+  } catch (error) {
+    throw new Error(
+      `The types for "${workspaceName}" do not exist.\n` +
+        `Please build types for workspace "${workspaceDir}" before running downlevel-dts script.`
+    );
+  }
+
+  const downlevelDir = join(declarationDir, downlevelDirname);
+  // Create downlevel-dts folder if it doesn't exist
+  try {
+    await access(downlevelDir);
+  } catch (error) {
+    await execPromise(["yarn", "downlevel-dts"].join(" "), { cwd: workspaceDir });
+  }
+
+  // Strip comments from downlevel-dts files
+  try {
+    await access(downlevelDir);
+    const files = await getAllFiles(downlevelDir);
+    for (const downlevelTypesFilepath of files) {
+      try {
+        const content = await readFile(downlevelTypesFilepath, "utf8");
+        await writeFile(downlevelTypesFilepath, stripComments(content));
+      } catch (error) {
+        console.error(`Error while stripping comments from "${downlevelTypesFilepath.replace(process.cwd(), "")}"`);
+        console.error(error);
+      }
+    }
+  } catch (error) {
+    // downlevelDir is not present, do nothing.
+  }
+};

scripts/downlevel-dts/getAllFiles.mjs

@@ -0,0 +1,17 @@
+import { readdir, stat } from "fs/promises";
+
+export const getAllFiles = async (dirPath, arrayOfFiles = []) => {
+  const files = await readdir(dirPath);
+
+  for (const file of files) {
+    const { isDirectory } = await stat(dirPath + "/" + file);
+    if (isDirectory()) {
+      const filesInDirectory = await getAllFiles(dirPath + "/" + file, arrayOfFiles);
+      arrayOfFiles.push(filesInDirectory);
+    } else {
+      arrayOfFiles.push(join(dirPath, "/", file));
+    }
+  }
+
+  return arrayOfFiles;
+};

scripts/downlevel-dts/getDeclarationDirname.mjs

@@ -0,0 +1,14 @@
+import { readFile } from "fs/promises";
+import { join } from "path";
+
+export const getDeclarationDirname = async (workspaceDir) => {
+  const tsTypesConfigPath = join(workspaceDir, "tsconfig.types.json");
+  const tsTypesConfigJson = JSON.parse((await readFile(tsTypesConfigPath)).toString());
+
+  const declarationDirname = tsTypesConfigJson.compilerOptions.declarationDir;
+  if (!declarationDirname) {
+    throw new Error(`The declarationDir is not defined in "${tsTypesConfigPath}".`);
+  }
+
+  return declarationDirname;
+};

scripts/downlevel-dts/getDownlevelDirname.mjs

@@ -0,0 +1,13 @@
+import { readFile } from "fs/promises";
+import { join } from "path";
+
+export const getDownlevelDirname = async (workspaceDir) => {
+  const packageJsonPath = join(workspaceDir, "package.json");
+  const packageJson = JSON.parse((await readFile(packageJsonPath)).toString());
+  if (!packageJson.scripts["downlevel-dts"]) {
+    console.error(`The "downlevel-dts" script is not defined for "${workspaceDir}"`);
+    return;
+  }
+  const downlevelArgs = packageJson.scripts["downlevel-dts"].split(" ");
+  return downlevelArgs[2].replace(`${downlevelArgs[1]}/`, "");
+};

scripts/downlevel-dts/getWorkspaces.mjs

@@ -0,0 +1,13 @@
+import { readdirSync, readFileSync } from "fs";
+import { join } from "path";
+
+export const getWorkspaces = (rootDir) => {
+  const { packages } = JSON.parse(readFileSync(join(rootDir, "package.json")).toString()).workspaces;
+  return packages
+    .map((dir) => dir.replace("/*", ""))
+    .flatMap((workspacesDir) =>
+      readdirSync(join(rootDir, workspacesDir), { withFileTypes: true })
+        .filter((dirent) => dirent.isDirectory())
+        .map((dirent) => ({ workspacesDir, workspaceName: dirent.name }))
+    );
+};

scripts/downlevel-dts/index.js

@@ -0,0 +1,28 @@
+// @ts-check
+import parallelLimit from "async/parallelLimit";
+import { cpus } from "os";
+import yargs from "yargs";
+
+import { downlevelWorkspace } from "./downlevelWorkspace.mjs";
+import { getWorkspaces } from "./getWorkspaces.mjs";
+
+// ToDo: Write downlevel-dts as a yargs command, and import yargs in scripts instead.
+yargs
+  .usage(
+    "Runs downlevel-dts npm script (if present) in each workspace of monorepo," +
+      " and strips comments from *.d.ts files.\n\n" +
+      "Usage: index.mjs"
+  )
+  .help()
+  .alias("h", "help").argv;
+
+const workspaces = getWorkspaces(process.cwd());
+const tasks = workspaces.map(({ workspacesDir, workspaceName }) => async () => {
+  await downlevelWorkspace(workspacesDir, workspaceName);
+});
+
+parallelLimit(tasks, cpus().length, function (err) {
+  if (err) {
+    throw err;
+  }
+});

scripts/downlevel-dts/index.mjs

@@ -2601,6 +2601,11 @@ [email protected], async@^3.0.1:
   resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720"
   integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==
 
+[email protected]:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.2.1.tgz#d3274ec66d107a47476a4c49136aacdb00665fc8"
+  integrity sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg==
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"