refactor(@angular-devkit/build-angular): reorganize bundle processing for browser builder (#15776)

clydin · mgechev · commit 2dc885304bc0 · 2019-10-09T11:25:23.000-07:00
diff --git a/packages/angular_devkit/build_angular/src/browser/action-cache.ts b/packages/angular_devkit/build_angular/src/browser/action-cache.ts
@@ -0,0 +1,191 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { createHash } from 'crypto';
+import * as findCacheDirectory from 'find-cache-dir';
+import * as fs from 'fs';
+import { manglingDisabled } from '../utils/mangle-options';
+import { CacheKey, ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle';
+
+const cacache = require('cacache');
+const cacheDownlevelPath = findCacheDirectory({ name: 'angular-build-dl' });
+const packageVersion = require('../../package.json').version;
+
+// Workaround Node.js issue prior to 10.16 with copyFile on macOS
+// https://github.com/angular/angular-cli/issues/15544 & https://github.com/nodejs/node/pull/27241
+let copyFileWorkaround = false;
+if (process.platform === 'darwin') {
+  const version = process.versions.node.split('.').map(part => Number(part));
+  if (version[0] < 10 || version[0] === 11 || (version[0] === 10 && version[1] < 16)) {
+    copyFileWorkaround = true;
+  }
+}
+
+export interface CacheEntry {
+  path: string;
+  size: number;
+  integrity?: string;
+}
+
+export class BundleActionCache {
+  constructor(private readonly integrityAlgorithm?: string) {}
+
+  static copyEntryContent(entry: CacheEntry | string, dest: fs.PathLike): void {
+    if (copyFileWorkaround) {
+      try {
+        fs.unlinkSync(dest);
+      } catch {}
+    }
+
+    fs.copyFileSync(
+      typeof entry === 'string' ? entry : entry.path,
+      dest,
+      fs.constants.COPYFILE_FICLONE,
+    );
+    if (process.platform !== 'win32') {
+      // The cache writes entries as readonly and when using copyFile the permissions will also be copied.
+      // See: https://github.com/npm/cacache/blob/073fbe1a9f789ba42d9a41de7b8429c93cf61579/lib/util/move-file.js#L36
+      fs.chmodSync(dest, 0o644);
+    }
+  }
+
+  generateBaseCacheKey(content: string): string {
+    // Create base cache key with elements:
+    // * package version - different build-angular versions cause different final outputs
+    // * code length/hash - ensure cached version matches the same input code
+    const algorithm = this.integrityAlgorithm || 'sha1';
+    const codeHash = createHash(algorithm)
+      .update(content)
+      .digest('base64');
+    let baseCacheKey = `${packageVersion}|${content.length}|${algorithm}-${codeHash}`;
+    if (manglingDisabled) {
+      baseCacheKey += '|MD';
+    }
+
+    return baseCacheKey;
+  }
+
+  generateCacheKeys(action: ProcessBundleOptions): string[] {
+    const baseCacheKey = this.generateBaseCacheKey(action.code);
+
+    // Postfix added to sourcemap cache keys when vendor sourcemaps are present
+    // Allows non-destructive caching of both variants
+    const SourceMapVendorPostfix = !!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : '';
+
+    // Determine cache entries required based on build settings
+    const cacheKeys = [];
+
+    // If optimizing and the original is not ignored, add original as required
+    if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) {
+      cacheKeys[CacheKey.OriginalCode] = baseCacheKey + '|orig';
+
+      // If sourcemaps are enabled, add original sourcemap as required
+      if (action.sourceMaps) {
+        cacheKeys[CacheKey.OriginalMap] = baseCacheKey + SourceMapVendorPostfix + '|orig-map';
+      }
+    }
+    // If not only optimizing, add downlevel as required
+    if (!action.optimizeOnly) {
+      cacheKeys[CacheKey.DownlevelCode] = baseCacheKey + '|dl';
+
+      // If sourcemaps are enabled, add downlevel sourcemap as required
+      if (action.sourceMaps) {
+        cacheKeys[CacheKey.DownlevelMap] = baseCacheKey + SourceMapVendorPostfix + '|dl-map';
+      }
+    }
+
+    return cacheKeys;
+  }
+
+  async getCacheEntries(cacheKeys: (string | null)[]): Promise<(CacheEntry | null)[] | false> {
+    // Attempt to get required cache entries
+    const cacheEntries = [];
+    for (const key of cacheKeys) {
+      if (key) {
+        const entry = await cacache.get.info(cacheDownlevelPath, key);
+        if (!entry) {
+          return false;
+        }
+        cacheEntries.push({
+          path: entry.path,
+          size: entry.size,
+          integrity: entry.metadata && entry.metadata.integrity,
+        });
+      } else {
+        cacheEntries.push(null);
+      }
+    }
+
+    return cacheEntries;
+  }
+
+  async getCachedBundleResult(action: ProcessBundleOptions): Promise<ProcessBundleResult | null> {
+    const entries = action.cacheKeys && await this.getCacheEntries(action.cacheKeys);
+    if (!entries) {
+      return null;
+    }
+
+    const result: ProcessBundleResult = { name: action.name };
+
+    let cacheEntry = entries[CacheKey.OriginalCode];
+    if (cacheEntry) {
+      result.original = {
+        filename: action.filename,
+        size: cacheEntry.size,
+        integrity: cacheEntry.integrity,
+      };
+
+      BundleActionCache.copyEntryContent(cacheEntry, result.original.filename);
+
+      cacheEntry = entries[CacheKey.OriginalMap];
+      if (cacheEntry) {
+        result.original.map = {
+          filename: action.filename + '.map',
+          size: cacheEntry.size,
+        };
+
+        BundleActionCache.copyEntryContent(cacheEntry, result.original.filename + '.map');
+      }
+    } else if (!action.ignoreOriginal) {
+      // If the original wasn't processed (and therefore not cached), add info
+      result.original = {
+        filename: action.filename,
+        size: Buffer.byteLength(action.code, 'utf8'),
+        map:
+          action.map === undefined
+            ? undefined
+            : {
+                filename: action.filename + '.map',
+                size: Buffer.byteLength(action.map, 'utf8'),
+              },
+      };
+    }
+
+    cacheEntry = entries[CacheKey.DownlevelCode];
+    if (cacheEntry) {
+      result.downlevel = {
+        filename: action.filename.replace('es2015', 'es5'),
+        size: cacheEntry.size,
+        integrity: cacheEntry.integrity,
+      };
+
+      BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename);
+
+      cacheEntry = entries[CacheKey.DownlevelMap];
+      if (cacheEntry) {
+        result.downlevel.map = {
+          filename: action.filename.replace('es2015', 'es5') + '.map',
+          size: cacheEntry.size,
+        };
+
+        BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename + '.map');
+      }
+    }
+
+    return result;
+  }
+}
diff --git a/packages/angular_devkit/build_angular/src/browser/action-executor.ts b/packages/angular_devkit/build_angular/src/browser/action-executor.ts
@@ -7,49 +7,110 @@
  */
 import JestWorker from 'jest-worker';
 import * as os from 'os';
+import * as path from 'path';
+import { ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle';
+import { BundleActionCache } from './action-cache';
 
-export class ActionExecutor<Input extends { size: number }, Output> {
-  private largeWorker: JestWorker;
-  private smallWorker: JestWorker;
+let workerFile = require.resolve('../utils/process-bundle');
+workerFile =
+  path.extname(workerFile) === '.ts'
+    ? require.resolve('../utils/process-bundle-bootstrap')
+    : workerFile;
 
-  private smallThreshold = 32 * 1024;
+export class BundleActionExecutor {
+  private largeWorker?: JestWorker;
+  private smallWorker?: JestWorker;
+  private cache: BundleActionCache;
+
+  constructor(
+    private workerOptions: unknown,
+    integrityAlgorithm?: string,
+    private readonly sizeThreshold = 32 * 1024,
+  ) {
+    this.cache = new BundleActionCache(integrityAlgorithm);
+  }
+
+  private static executeMethod<O>(worker: JestWorker, method: string, input: unknown): Promise<O> {
+    return ((worker as unknown) as Record<string, (i: unknown) => Promise<O>>)[method](input);
+  }
+
+  private ensureLarge(): JestWorker {
+    if (this.largeWorker) {
+      return this.largeWorker;
+    }
 
-  constructor(actionFile: string, private readonly actionName: string, setupOptions?: unknown) {
     // larger files are processed in a separate process to limit memory usage in the main process
-    this.largeWorker = new JestWorker(actionFile, {
-      exposedMethods: [actionName],
-      setupArgs: setupOptions === undefined ? undefined : [setupOptions],
-    });
+    return (this.largeWorker = new JestWorker(workerFile, {
+      exposedMethods: ['process'],
+      setupArgs: [this.workerOptions],
+    }));
+  }
+
+  private ensureSmall(): JestWorker {
+    if (this.smallWorker) {
+      return this.smallWorker;
+    }
 
     // small files are processed in a limited number of threads to improve speed
     // The limited number also prevents a large increase in memory usage for an otherwise short operation
-    this.smallWorker = new JestWorker(actionFile, {
-      exposedMethods: [actionName],
-      setupArgs: setupOptions === undefined ? undefined : [setupOptions],
+    return (this.smallWorker = new JestWorker(workerFile, {
+      exposedMethods: ['process'],
+      setupArgs: [this.workerOptions],
       numWorkers: os.cpus().length < 2 ? 1 : 2,
       // Will automatically fallback to processes if not supported
       enableWorkerThreads: true,
-    });
+    }));
   }
 
-  execute(options: Input): Promise<Output> {
-    if (options.size > this.smallThreshold) {
-      return ((this.largeWorker as unknown) as Record<string, (options: Input) => Promise<Output>>)[
-        this.actionName
-      ](options);
+  private executeAction<O>(method: string, action: { code: string }): Promise<O> {
+    // code.length is not an exact byte count but close enough for this
+    if (action.code.length > this.sizeThreshold) {
+      return BundleActionExecutor.executeMethod<O>(this.ensureLarge(), method, action);
     } else {
-      return ((this.smallWorker as unknown) as Record<string, (options: Input) => Promise<Output>>)[
-        this.actionName
-      ](options);
+      return BundleActionExecutor.executeMethod<O>(this.ensureSmall(), method, action);
     }
   }
 
-  executeAll(options: Input[]): Promise<Output[]> {
-    return Promise.all(options.map(o => this.execute(o)));
+  async process(action: ProcessBundleOptions) {
+    const cacheKeys = this.cache.generateCacheKeys(action);
+    action.cacheKeys = cacheKeys;
+
+    // Try to get cached data, if it fails fallback to processing
+    try {
+      const cachedResult = await this.cache.getCachedBundleResult(action);
+      if (cachedResult) {
+        return cachedResult;
+      }
+    } catch {}
+
+    return this.executeAction<ProcessBundleResult>('process', action);
+  }
+
+  async *processAll(actions: Iterable<ProcessBundleOptions>) {
+    const executions = new Map<Promise<ProcessBundleResult>, Promise<ProcessBundleResult>>();
+    for (const action of actions) {
+      const execution = this.process(action);
+      executions.set(
+        execution,
+        execution.then(result => {
+          executions.delete(execution);
+
+          return result;
+        }),
+      );
+    }
+
+    while (executions.size > 0) {
+      yield Promise.race(executions.values());
+    }
   }
 
   stop() {
-    this.largeWorker.end();
-    this.smallWorker.end();
+    if (this.largeWorker) {
+      this.largeWorker.end();
+    }
+    if (this.smallWorker) {
+      this.smallWorker.end();
+    }
   }
 }
diff --git a/packages/angular_devkit/build_angular/src/browser/index.ts b/packages/angular_devkit/build_angular/src/browser/index.ts
