feat: use new read/load assembly functions everywhere (#3600)

This PR utilizes the functions added in #3570 everywhere in the jsii monorepo. The benefit of this is two-fold -- it refactors all places where we load/read assemblies and points them to the source of truth (introduced in #3570). This also means the logic for compressing assemblies will be available for all packages in the jsii monorepo if/when we flip that switch.

---

By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license].

[Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0

Author: kaizencc

packages/@jsii/kernel/src/kernel.ts

@@ -1,4 +1,5 @@
 import * as spec from '@jsii/spec';
+import { loadAssemblyFromPath } from '@jsii/spec';
 import * as cp from 'child_process';
 import * as fs from 'fs-extra';
 import * as os from 'os';
@@ -114,13 +115,12 @@ export class Kernel {
     }
 
     // read .jsii metadata from the root of the package
-    const jsiiMetadataFile = path.join(packageDir, spec.SPEC_FILE_NAME);
-    if (!fs.pathExistsSync(jsiiMetadataFile)) {
-      throw new Error(
-        `Package tarball ${req.tarball} must have a file named ${spec.SPEC_FILE_NAME} at the root`,
-      );
+    let assmSpec;
+    try {
+      assmSpec = loadAssemblyFromPath(packageDir);
+    } catch (e: any) {
+      throw new Error(`Error for package tarball ${req.tarball}: ${e.message}`);
     }
-    const assmSpec = fs.readJsonSync(jsiiMetadataFile) as spec.Assembly;
 
     // load the module and capture it's closure
     const closure = this._execute(

packages/@jsii/runtime/test/kernel-host.test.ts

@@ -1,5 +1,6 @@
 import { api } from '@jsii/kernel';
 import * as spec from '@jsii/spec';
+import { loadAssemblyFromPath } from '@jsii/spec';
 import * as child from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -86,12 +87,9 @@ function loadRequest(library: string): api.LoadRequest {
   };
 
   function loadAssembly(): spec.Assembly {
-    const assemblyFile = path.resolve(
-      require.resolve(`${library}/package.json`),
-      '..',
-      '.jsii',
+    return loadAssemblyFromPath(
+      path.resolve(require.resolve(`${library}/package.json`), '..'),
     );
-    return JSON.parse(fs.readFileSync(assemblyFile, { encoding: 'utf-8' }));
   }
 
   function packageLibrary(target: string): void {

packages/@scope/jsii-calc-base-of-base/.npmignore

@@ -4,8 +4,9 @@
 *.tgz
 
 
-# Include .jsii
+# Include .jsii and .jsii.gz
 !.jsii
+!.jsii.gz
 
 
 # Exclude jsii outdir

packages/@scope/jsii-calc-base/.npmignore

@@ -4,8 +4,9 @@
 *.tgz
 
 
-# Include .jsii
+# Include .jsii and .jsii.gz
 !.jsii
+!.jsii.gz
 
 
 # Exclude jsii outdir

packages/@scope/jsii-calc-lib/.npmignore

@@ -4,8 +4,9 @@
 *.tgz
 
 
-# Include .jsii
+# Include .jsii and .jsii.gz
 !.jsii
+!.jsii.gz
 
 
 # Exclude jsii outdir

packages/jsii-calc/.npmignore

@@ -3,8 +3,9 @@
 !*.d.ts
 *.tgz
 
-# Include .jsii
+# Include .jsii and .jsii.gz
 !.jsii
+!.jsii.gz
 
 
 # Exclude jsii outdir

packages/jsii-diff/bin/jsii-diff.ts

@@ -215,7 +215,7 @@ type LoadAssemblyResult = { requested: string; resolved: string } & (
 async function loadPackageNameFromAssembly(
   options: LoadOptions,
 ): Promise<string> {
-  const JSII_ASSEMBLY_FILE = '.jsii';
+  const JSII_ASSEMBLY_FILE = spec.SPEC_FILE_NAME;
   if (!(await fs.pathExists(JSII_ASSEMBLY_FILE))) {
     throw new Error(
       `No NPM package name given and no ${JSII_ASSEMBLY_FILE} file in the current directory. Please specify a package name.`,

packages/jsii-pacmak/lib/npm-modules.ts

@@ -150,7 +150,11 @@ async function updateNpmIgnore(
     );
   }
 
-  includePattern('Include .jsii', spec.SPEC_FILE_NAME);
+  includePattern(
+    'Include .jsii and .jsii.gz',
+    spec.SPEC_FILE_NAME,
+    spec.SPEC_FILE_NAME_COMPRESSED,
+  );
 
   if (modified) {
     await fs.writeFile(npmIgnorePath, `${lines.join('\n')}\n`);

packages/jsii-reflect/lib/type-system.ts

@@ -1,4 +1,4 @@
-import * as jsii from '@jsii/spec';
+import { getAssemblyFile, loadAssemblyFromFile } from '@jsii/spec';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
@@ -109,10 +109,7 @@ export class TypeSystem {
       // Load the assembly, but don't recurse if we already have an assembly with the same name.
       // Validation is not an insignificant time sink, and loading IS insignificant, so do a
       // load without validation first. This saves about 2/3rds of processing time.
-      const asm = await this.loadAssembly(
-        path.join(moduleDirectory, '.jsii'),
-        false,
-      );
+      const asm = this.loadAssembly(getAssemblyFile(moduleDirectory), false);
       if (this.includesAssembly(asm.name)) {
         const existing = this.findAssembly(asm.name);
         if (existing.version !== asm.version) {
@@ -154,11 +151,11 @@ export class TypeSystem {
     }
   }
 
-  public async loadFile(
+  public loadFile(
     file: string,
     options: { isRoot?: boolean; validate?: boolean } = {},
   ) {
-    const assembly = await this.loadAssembly(file, options.validate !== false);
+    const assembly = this.loadAssembly(file, options.validate !== false);
     return this.addAssembly(assembly, options);
   }
 
@@ -308,12 +305,9 @@ export class TypeSystem {
    * @param file Assembly file to load
    * @param validate Whether to validate the assembly or just assume it matches the schema
    */
-  private async loadAssembly(file: string, validate = true) {
-    const spec = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' }));
-    const ass = validate
-      ? jsii.validateAssembly(spec)
-      : (spec as jsii.Assembly);
-    return new Assembly(this, ass);
+  private loadAssembly(file: string, validate = true) {
+    const contents = loadAssemblyFromFile(file, validate);
+    return new Assembly(this, contents);
   }
 
   private addRoot(asm: Assembly) {

packages/jsii-rosetta/lib/commands/coverage.ts

@@ -5,7 +5,7 @@ import { formatLocation } from '../snippet';
 
 export async function checkCoverage(assemblyLocations: readonly string[]): Promise<void> {
   logging.info(`Loading ${assemblyLocations.length} assemblies`);
-  const assemblies = await loadAssemblies(assemblyLocations, false);
+  const assemblies = loadAssemblies(assemblyLocations, false);
 
   const snippets = Array.from(await allTypeScriptSnippets(assemblies, true));
 

packages/jsii-rosetta/lib/commands/extract.ts

@@ -94,7 +94,7 @@ export async function extractSnippets(
   const only = options.only ?? [];
 
   logging.info(`Loading ${assemblyLocations.length} assemblies`);
-  const assemblies = await loadAssemblies(assemblyLocations, options.validateAssemblies ?? false);
+  const assemblies = loadAssemblies(assemblyLocations, options.validateAssemblies ?? false);
 
   let snippets = Array.from(await allTypeScriptSnippets(assemblies, options.loose));
   if (only.length > 0) {
@@ -170,7 +170,7 @@ export async function extractSnippets(
     const output = options.trimCache
       ? new LanguageTablet()
       : await LanguageTablet.fromOptionalFile(options.cacheToFile);
-    output.addTablet(translator.tablet);
+    output.addTablets(translator.tablet);
     await output.save(options.cacheToFile);
   }
 

packages/jsii-rosetta/lib/commands/infuse.ts

@@ -66,7 +66,7 @@ export async function infuse(assemblyLocations: string[], options?: InfuseOption
   }
 
   // Load tablet file and assemblies
-  const assemblies = await loadAssemblies(assemblyLocations, false);
+  const assemblies = loadAssemblies(assemblyLocations, false);
   const defaultTablets = await loadAllDefaultTablets(assemblies);
 
   const availableTranslations = new LanguageTablet();

packages/jsii-rosetta/lib/commands/transliterate.ts

@@ -1,5 +1,5 @@
-import { Assembly, Docs, SPEC_FILE_NAME, Type, TypeKind } from '@jsii/spec';
-import { readJson, writeJson } from 'fs-extra';
+import { Assembly, Docs, SPEC_FILE_NAME, Type, TypeKind, loadAssemblyFromPath } from '@jsii/spec';
+import { writeJson } from 'fs-extra';
 import { resolve } from 'path';
 
 import { TargetLanguage } from '../languages';
@@ -98,8 +98,7 @@ export async function transliterateAssembly(
   for (const [location, loadAssembly] of assemblies.entries()) {
     for (const language of targetLanguages) {
       const now = new Date().getTime();
-      // eslint-disable-next-line no-await-in-loop
-      const result = await loadAssembly();
+      const result = loadAssembly();
 
       if (result.targets?.[targetName(language)] == null) {
         // This language is not supported by the assembly, so we skip it...
@@ -149,16 +148,16 @@ async function loadAssemblies(
   const result = new Map<string, AssemblyLoader>();
 
   for (const directory of directories) {
-    const loader = () => readJson(resolve(directory, SPEC_FILE_NAME));
+    const loader = () => loadAssemblyFromPath(directory);
     // eslint-disable-next-line no-await-in-loop
-    await rosetta.addAssembly(await loader(), directory);
+    await rosetta.addAssembly(loader(), directory);
     result.set(directory, loader);
   }
 
   return result;
 }
 
-type AssemblyLoader = () => Promise<Mutable<Assembly>>;
+type AssemblyLoader = () => Mutable<Assembly>;
 
 function transliterateType(type: Type, rosetta: RosettaTabletReader, language: TargetLanguage): void {
   transliterateDocs({ api: 'type', fqn: type.fqn }, type.docs);

packages/jsii-rosetta/lib/commands/trim-cache.ts

@@ -18,7 +18,7 @@ export interface TrimCacheOptions {
 
 export async function trimCache(options: TrimCacheOptions): Promise<void> {
   logging.info(`Loading ${options.assemblyLocations.length} assemblies`);
-  const assemblies = await loadAssemblies(options.assemblyLocations, false);
+  const assemblies = loadAssemblies(options.assemblyLocations, false);
 
   const snippets = Array.from(await allTypeScriptSnippets(assemblies));
 

packages/jsii-rosetta/lib/jsii/assemblies.ts

@@ -1,4 +1,5 @@
 import * as spec from '@jsii/spec';
+import { loadAssemblyFromFile, loadAssemblyFromPath, getAssemblyFile, writeAssembly } from '@jsii/spec';
 import * as crypto from 'crypto';
 import * as fs from 'fs-extra';
 import * as path from 'path';
@@ -55,35 +56,28 @@ export interface LoadedAssembly {
 /**
  * Load assemblies by filename or directory
  */
-export async function loadAssemblies(
+export function loadAssemblies(
   assemblyLocations: readonly string[],
   validateAssemblies: boolean,
-): Promise<readonly LoadedAssembly[]> {
-  return Promise.all(assemblyLocations.map(loadAssembly));
+): readonly LoadedAssembly[] {
+  return assemblyLocations.map(loadAssembly);
 
-  async function loadAssembly(location: string): Promise<LoadedAssembly> {
-    const stat = await fs.stat(location);
+  function loadAssembly(location: string): LoadedAssembly {
+    const stat = fs.statSync(location);
     if (stat.isDirectory()) {
-      return loadAssembly(path.join(location, '.jsii'));
+      return loadAssembly(getAssemblyFile(location));
     }
 
     const directory = path.dirname(location);
     const pjLocation = path.join(directory, 'package.json');
 
-    const [assembly, packageJson] = await Promise.all([
-      loadAssemblyFromFile(location, validateAssemblies),
-      (await fs.pathExists(pjLocation)) ? fs.readJSON(pjLocation, { encoding: 'utf-8' }) : Promise.resolve(undefined),
-    ]);
+    const assembly = loadAssemblyFromFile(location, validateAssemblies);
+    const packageJson = fs.pathExistsSync(pjLocation) ? fs.readJSONSync(pjLocation, { encoding: 'utf-8' }) : undefined;
 
     return { assembly, directory, packageJson };
   }
 }
 
-async function loadAssemblyFromFile(filename: string, validate: boolean): Promise<spec.Assembly> {
-  const contents = await fs.readJSON(filename, { encoding: 'utf-8' });
-  return validate ? spec.validateAssembly(contents) : (contents as spec.Assembly);
-}
-
 /**
  * Load the default tablets for every assembly, if available
  *
@@ -236,12 +230,12 @@ export async function allTypeScriptSnippets(
  * Replaces the file where the original assembly file *should* be found with a new assembly file.
  * Recalculates the fingerprint of the assembly to avoid tampering detection.
  */
-export async function replaceAssembly(assembly: spec.Assembly, directory: string): Promise<void> {
-  const fileName = path.join(directory, '.jsii');
-  await fs.writeJson(fileName, _fingerprint(assembly), {
-    encoding: 'utf8',
-    spaces: 2,
-  });
+export function replaceAssembly(
+  assembly: spec.Assembly,
+  directory: string,
+  { compress = false }: { compress?: boolean } = {},
+) {
+  writeAssembly(directory, _fingerprint(assembly), { compress });
 }
 
 /**
@@ -296,24 +290,23 @@ export function findTypeLookupAssembly(startingDirectory: string): TypeLookupAss
 }
 
 function loadLookupAssembly(directory: string): TypeLookupAssembly | undefined {
-  const assemblyFile = path.join(directory, '.jsii');
-  if (!fs.pathExistsSync(assemblyFile)) {
+  try {
+    const packageJson = fs.readJSONSync(path.join(directory, 'package.json'), { encoding: 'utf-8' });
+    const assembly: spec.Assembly = loadAssemblyFromPath(directory);
+    const symbolIdMap = mkDict([
+      ...Object.values(assembly.types ?? {}).map((type) => [type.symbolId ?? '', type.fqn] as const),
+      ...Object.entries(assembly.submodules ?? {}).map(([fqn, mod]) => [mod.symbolId ?? '', fqn] as const),
+    ]);
+
+    return {
+      packageJson,
+      assembly,
+      directory,
+      symbolIdMap,
+    };
+  } catch {
     return undefined;
   }
-
-  const packageJson = fs.readJSONSync(path.join(directory, 'package.json'), { encoding: 'utf-8' });
-  const assembly: spec.Assembly = fs.readJSONSync(assemblyFile, { encoding: 'utf-8' });
-  const symbolIdMap = mkDict([
-    ...Object.values(assembly.types ?? {}).map((type) => [type.symbolId ?? '', type.fqn] as const),
-    ...Object.entries(assembly.submodules ?? {}).map(([fqn, mod]) => [mod.symbolId ?? '', fqn] as const),
-  ]);
-
-  return {
-    packageJson,
-    assembly,
-    directory,
-    symbolIdMap,
-  };
 }
 
 function findPackageJsonLocation(currentPath: string): string | undefined {