chore(core): create submodule exports in core (#6079)

* chore(core): create submodule exports in core

* chore(core): organize submodules

* chore(core): submodules linting

* chore(core): submodule readmes

* chore(core): update readme about Metro package exports support

* chore(core): add compatibility redirect files for core submodules

* chore(core): add files entry in pkg.json

* chore(core): add files pkg.json enforcement

* chore(core): import submodules in core root index

Author: kuhe

README.md

@@ -24,6 +24,9 @@
     "@smithy/url-parser": "^3.0.0",
     "tslib": "^2.6.2"
   },
+  "files": [
+    "dist-*/**"
+  ],
   "devDependencies": {
     "@tsconfig/recommended": "1.0.1",
     "concurrently": "7.0.0",

packages/cloudfront-signer/package.json

@@ -0,0 +1,5 @@
+/**
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
+ */
+module.exports = require("./dist-cjs/submodules/client/index.js");

packages/core/README.md

@@ -0,0 +1,5 @@
+/**
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
+ */
+module.exports = require("./dist-cjs/submodules/httpAuthSchemes/index.js");

packages/core/client.js

@@ -3,7 +3,7 @@
   "version": "3.575.0",
   "description": "Core functions & classes shared by multiple AWS SDK clients",
   "scripts": {
-    "build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
+    "build": "yarn lint && concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
     "build:cjs": "node ../../scripts/compilation/inline core",
     "build:es": "tsc -p tsconfig.es.json",
     "build:include:deps": "lerna run --scope $npm_package_name --include-dependencies build",
@@ -18,6 +18,43 @@
   "main": "./dist-cjs/index.js",
   "module": "./dist-es/index.js",
   "types": "./dist-types/index.d.ts",
+  "exports": {
+    ".": {
+      "node": "./dist-cjs/index.js",
+      "import": "./dist-es/index.js",
+      "require": "./dist-cjs/index.js",
+      "types": "./dist-types/index.d.ts"
+    },
+    "./package.json": {
+      "node": "./package.json",
+      "import": "./package.json",
+      "require": "./package.json"
+    },
+    "./client": {
+      "node": "./dist-cjs/submodules/client/index.js",
+      "import": "./dist-es/submodules/client/index.js",
+      "require": "./dist-cjs/submodules/client/index.js",
+      "types": "./dist-types/submodules/client/index.d.ts"
+    },
+    "./httpAuthSchemes": {
+      "node": "./dist-cjs/submodules/httpAuthSchemes/index.js",
+      "import": "./dist-es/submodules/httpAuthSchemes/index.js",
+      "require": "./dist-cjs/submodules/httpAuthSchemes/index.js",
+      "types": "./dist-types/submodules/httpAuthSchemes/index.d.ts"
+    },
+    "./protocols": {
+      "node": "./dist-cjs/submodules/protocols/index.js",
+      "import": "./dist-es/submodules/protocols/index.js",
+      "require": "./dist-cjs/submodules/protocols/index.js",
+      "types": "./dist-types/submodules/protocols/index.d.ts"
+    }
+  },
+  "files": [
+    "dist-*/**",
+    "./client.js",
+    "./httpAuthSchemes.js",
+    "./protocols.js"
+  ],
   "sideEffects": false,
   "author": {
     "name": "AWS SDK for JavaScript Team",

packages/core/httpAuthSchemes.js

@@ -0,0 +1,5 @@
+/**
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
+ */
+module.exports = require("./dist-cjs/submodules/protocols/index.js");

packages/core/package.json

@@ -1,24 +1,98 @@
 const fs = require("fs");
 const path = require("path");
-const assert = require("assert");
 
 const root = path.join(__dirname, "..");
 
 const pkgJson = require(path.join(root, "package.json"));
-const srcFolders = fs.readdirSync(path.join(root, "src"));
+const tsconfigs = {
+  cjs: require(path.join(root, "tsconfig.cjs.json")),
+  es: require(path.join(root, "tsconfig.es.json")),
+  types: require(path.join(root, "tsconfig.types.json")),
+};
+const submodules = fs.readdirSync(path.join(root, "src", "submodules"));
 
-assert(pkgJson.exports === undefined, "We cannot support package.json exports yet.");
+const errors = [];
 
+for (const submodule of submodules) {
+  const submodulePath = path.join(root, "src", "submodules", submodule);
+  if (fs.existsSync(submodulePath) && fs.lstatSync(submodulePath).isDirectory()) {
+    // package.json metadata.
+    if (!pkgJson.exports[`./${submodule}`]) {
+      errors.push(`${submodule} submodule is missing exports statement in package.json`);
+      pkgJson.exports[`./${submodule}`] = {
+        node: `./dist-cjs/submodules/${submodule}/index.js`,
+        import: `./dist-es/submodules/${submodule}/index.js`,
+        require: `./dist-cjs/submodules/${submodule}/index.js`,
+        types: `./dist-types/submodules/${submodule}/index.d.ts`,
+      };
+      fs.writeFileSync(path.join(root, "package.json"), JSON.stringify(pkgJson, null, 2) + "\n");
+    }
+    if (!pkgJson.files.includes(`./${submodule}.js`)) {
+      pkgJson.files.push(`./${submodule}.js`);
+      errors.push(`package.json files array missing ${submodule}.js compatibility redirect file.`);
+      fs.writeFileSync(path.join(root, "package.json"), JSON.stringify(pkgJson, null, 2) + "\n");
+    }
+    // tsconfig metadata.
+    for (const [kind, tsconfig] of Object.entries(tsconfigs)) {
+      if (!tsconfig.compilerOptions.paths?.[`@aws-sdk/core/${submodule}`]) {
+        errors.push(`${submodule} submodule is missing paths entry in tsconfig.${kind}.json`);
+
+        tsconfig.compilerOptions.paths[`@aws-sdk/core/${submodule}`] = [`./src/submodules/${submodule}/index.ts`];
+        fs.writeFileSync(path.join(root, `tsconfig.${kind}.json`), JSON.stringify(tsconfig, null, 2) + "\n");
+      }
+    }
+    // compatibility redirect file.
+    const compatibilityRedirectFile = path.join(root, `${submodule}.js`);
+    if (!fs.existsSync(compatibilityRedirectFile)) {
+      errors.push(`${submodule} is missing compatibility redirect file in the package root folder.`);
+      fs.writeFileSync(
+        compatibilityRedirectFile,
+        `
 /**
- * We probably can't enable package.json exports until
- * dropping support for Node.js 14.x and TypeScript 4.6.
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
  */
-process.exit(0);
-
-for (const srcFolder of srcFolders) {
-  if (fs.lstatSync(path.join(root, "src", srcFolder)).isDirectory()) {
-    if (!pkgJson.exports["./" + srcFolder]) {
-      throw new Error(`${srcFolder} is missing exports statement in package.json`);
+module.exports = require("./dist-cjs/submodules/${submodule}/index.js");
+`
+      );
     }
   }
 }
+
+/**
+ * Check for cross-submodule relative imports.
+ */
+
+const walk = require("../../../scripts/utils/walk");
+
+(async () => {
+  for await (const item of walk(path.join(root, "src", "submodules"))) {
+    // depth within the submodule where 1 is at the root of the submodule.
+    const depth = item.split("core/src/submodules/")[1].split("/").length - 1;
+    const sourceCode = fs.readFileSync(item, "utf-8");
+
+    const relativeImports = [];
+    relativeImports.push(
+      ...new Set(
+        [...(sourceCode.toString().match(/(from |import\()"(.*?)";/g) || [])]
+          .map((_) => _.replace(/from "/g, "").replace(/";$/, ""))
+          .filter((_) => _.startsWith("."))
+      )
+    );
+
+    for (const i of relativeImports) {
+      const relativeImportDepth = i.split("..").length - 1;
+      if (relativeImportDepth >= depth) {
+        errors.push(
+          `relative import ${i} in ${item
+            .split("packages/")
+            .pop()} crosses submodule boundaries. Use @scope/package/submodule import instead.`
+        );
+      }
+    }
+  }
+})().then(() => {
+  if (errors.length) {
+    throw new Error(errors.join("\n"));
+  }
+});

packages/core/protocols.js

@@ -1,3 +1,24 @@
-export * from "./client/index";
-export * from "./httpAuthSchemes/index";
-export * from "./protocols/index";
+/**
+ * Submodules annotated with "Legacy" are from prior to the submodule system.
+ * They are exported from the package's root index to preserve backwards compatibility.
+ *
+ * New development should go in a proper submodule and not be exported from the root index.
+ */
+
+/**
+ * Legacy submodule.
+ */
+export * from "./submodules/client/index";
+/**
+ * Legacy submodule.
+ */
+export * from "./submodules/httpAuthSchemes/index";
+/**
+ * Legacy submodule.
+ */
+export * from "./submodules/protocols/index";
+
+/**
+ * Warning: do not export any additional submodules from the root of this package. See readme.md for
+ * guide on developing submodules.
+ */

packages/core/scripts/lint.js

@@ -2,7 +2,12 @@
   "compilerOptions": {
     "rootDir": "./src",
     "outDir": "./dist-cjs",
-    "baseUrl": "."
+    "baseUrl": ".",
+    "paths": {
+      "@aws-sdk/core/client": ["./src/submodules/client/index.ts"],
+      "@aws-sdk/core/httpAuthSchemes": ["./src/submodules/httpAuthSchemes/index.ts"],
+      "@aws-sdk/core/protocols": ["./src/submodules/protocols/index.ts"]
+    }
   },
   "extends": "../../tsconfig.cjs.json",
   "include": ["src/"]

packages/core/src/index.ts

@@ -2,7 +2,12 @@
   "compilerOptions": {
     "rootDir": "./src",
     "outDir": "./dist-es",
-    "baseUrl": "."
+    "baseUrl": ".",
+    "paths": {
+      "@aws-sdk/core/client": ["./src/submodules/client/index.ts"],
+      "@aws-sdk/core/httpAuthSchemes": ["./src/submodules/httpAuthSchemes/index.ts"],
+      "@aws-sdk/core/protocols": ["./src/submodules/protocols/index.ts"]
+    }
   },
   "extends": "../../tsconfig.es.json",
   "include": ["src/"]

packages/core/src/submodules/client/README.md

@@ -2,7 +2,12 @@
   "compilerOptions": {
     "baseUrl": ".",
     "declarationDir": "dist-types",
-    "rootDir": "src"
+    "rootDir": "src",
+    "paths": {
+      "@aws-sdk/core/client": ["./src/submodules/client/index.ts"],
+      "@aws-sdk/core/httpAuthSchemes": ["./src/submodules/httpAuthSchemes/index.ts"],
+      "@aws-sdk/core/protocols": ["./src/submodules/protocols/index.ts"]
+    }
   },
   "extends": "../../tsconfig.types.json",
   "include": ["src/"]

packages/core/src/client/emitWarningIfUnsupportedVersion.spec.ts renamed to packages/core/src/submodules/client/emitWarningIfUnsupportedVersion.spec.ts

@@ -19,6 +19,7 @@ module.exports = class Inliner {
     this.isPackage = fs.existsSync(path.join(root, "packages", pkg));
     this.isLib = fs.existsSync(path.join(root, "lib", pkg));
     this.isClient = !this.isPackage && !this.isLib;
+    this.isCore = pkg === "core";
     this.subfolder = this.isPackage ? "packages" : this.isLib ? "lib" : "clients";
 
     this.packageDirectory = path.join(root, this.subfolder, pkg);
@@ -149,9 +150,9 @@ module.exports = class Inliner {
       (variant) => "*/" + path.basename(variant).replace(/.js$/, "")
     );
 
-    await esbuild.build({
+    const buildOptions = {
       platform: this.platform,
-      target: ["node14"],
+      target: ["node16"],
       bundle: true,
       format: "cjs",
       mainFields: ["main"],
@@ -164,7 +165,27 @@ module.exports = class Inliner {
       keepNames: true,
       packages: "external",
       external: ["@smithy/*", "@aws-sdk/*", "node_modules/*", ...this.variantExternalsForEsBuild],
-    });
+    };
+
+    if (!this.isCore) {
+      await esbuild.build(buildOptions);
+    }
+
+    if (this.isCore) {
+      const submodules = fs.readdirSync(path.join(root, this.subfolder, this.package, "src", "submodules"));
+      for (const submodule of submodules) {
+        fs.rmSync(path.join(path.join(root, this.subfolder, this.package, "dist-cjs", "submodules", submodule)), {
+          recursive: true,
+          force: true,
+        });
+        await esbuild.build({
+          ...buildOptions,
+          entryPoints: [path.join(root, this.subfolder, this.package, "src", "submodules", submodule, "index.ts")],
+          outfile: path.join(root, this.subfolder, this.package, "dist-cjs", "submodules", submodule, "index.js"),
+        });
+      }
+    }
+
     return this;
   }
 
@@ -173,7 +194,7 @@ module.exports = class Inliner {
    * These now become re-exports of the index to preserve deep-import behavior.
    */
   async rewriteStubs() {
-    if (this.bailout) {
+    if (this.bailout || this.isCore) {
       return this;
     }
 

packages/core/src/client/emitWarningIfUnsupportedVersion.ts renamed to packages/core/src/submodules/client/emitWarningIfUnsupportedVersion.ts

@@ -55,6 +55,10 @@ module.exports = function (pkgJsonFilePath, overwrite = false) {
     errors.push(`browser and react-native fields are different in ${pkgJson.name}`);
   }
 
+  if (!pkgJson.files) {
+    errors.push(`no files entry in ${pkgJson.name}`);
+  }
+
   if (typeof pkgJson.browser === "object" && typeof pkgJson["react-native"] === "object") {
     const browserCanonical = Object.entries(pkgJson.browser).reduce((acc, [k, v]) => {
       if (!k.includes("dist-cjs/") || typeof v === "boolean") {