feat(ivy): find all packages to be compiled by ngcc (angular#25406)

PR Close angular#25406

Author: petebacondarwin

integration/ngcc/test.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 
-set -x
+set -e -x
 
 PATH=$PATH:$(npm bin)
 
-ivy-ngcc node_modules/@angular/common
-cp -r node_modules_ngtsc/* node_modules/
+ivy-ngcc fesm2015,esm2015
 ngc -p tsconfig-app.json
 
+# Look for correct output
+grep "directives: \[\S*\.NgIf\]" dist/src/main.js > /dev/null

packages/compiler-cli/src/ngcc/src/main.ts

@@ -5,23 +5,87 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
-import {resolve} from 'path';
+import {existsSync, lstatSync, readFileSync, readdirSync} from 'fs';
+import {posix as path} from 'path';
+
 import {PackageTransformer} from './transform/package_transformer';
 
 export function mainNgcc(args: string[]): number {
-  const packagePaths = args[0] ? [resolve(args[0])] : [];
-  const formats = args[1] ? [args[1]] : ['fesm2015', 'esm2015', 'fesm5', 'esm5'];
-
-  // TODO: find all the package types to transform
-  // TODO: error/warning logging/handling etc
+  const formats = args[0] ? args[0].split(',') : ['fesm2015', 'esm2015', 'fesm5', 'esm5'];
+  const packagePaths = args[1] ? [path.resolve(args[1])] : findPackagesToCompile();
+  const targetPath = args[2] ? args[2] : 'node_modules';
 
   const transformer = new PackageTransformer();
   packagePaths.forEach(packagePath => {
     formats.forEach(format => {
-      console.warn(`Compiling ${packagePath}:${format}`);
-      transformer.transform(packagePath, format);
+      // TODO: remove before flight
+      console.warn(`Compiling ${packagePath} : ${format}`);
+      transformer.transform(packagePath, format, targetPath);
     });
   });
 
   return 0;
 }
+
+// TODO - consider nested node_modules
+
+/**
+ * Check whether the given folder needs to be included in the ngcc compilation.
+ * We do not care about folders that are:
+ *
+ * - symlinks
+ * - node_modules
+ * - do not contain a package.json
+ * - do not have a typings property in package.json
+ * - do not have an appropriate metadata.json file
+ *
+ * @param folderPath The absolute path to the folder.
+ */
+function hasMetadataFile(folderPath: string): boolean {
+  const folderName = path.basename(folderPath);
+  if (folderName === 'node_modules' || lstatSync(folderPath).isSymbolicLink()) {
+    return false;
+  }
+  const packageJsonPath = path.join(folderPath, 'package.json');
+  if (!existsSync(packageJsonPath)) {
+    return false;
+  }
+  const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8'));
+  if (!packageJson.typings) {
+    return false;
+  }
+  // TODO: avoid if packageJson contains built marker
+  const metadataPath =
+      path.join(folderPath, packageJson.typings.replace(/\.d\.ts$/, '.metadata.json'));
+  return existsSync(metadataPath);
+}
+
+/**
+ * Look for packages that need to be compiled.
+ * The function will recurse into folders that start with `@...`, e.g. `@angular/...`.
+ * Without an argument it starts at `node_modules`.
+ */
+function findPackagesToCompile(folder: string = 'node_modules'): string[] {
+  const fullPath = path.resolve(folder);
+  const packagesToCompile: string[] = [];
+  readdirSync(fullPath)
+      .filter(p => !p.startsWith('.'))
+      .filter(p => lstatSync(path.join(fullPath, p)).isDirectory())
+      .forEach(p => {
+        const packagePath = path.join(fullPath, p);
+        if (p.startsWith('@')) {
+          packagesToCompile.push(...findPackagesToCompile(packagePath));
+        } else {
+          packagesToCompile.push(packagePath);
+        }
+      });
+
+  return packagesToCompile.filter(path => recursiveDirTest(path, hasMetadataFile));
+}
+
+function recursiveDirTest(dir: string, test: (dir: string) => boolean): boolean {
+  return test(dir) || readdirSync(dir).some(segment => {
+    const fullPath = path.join(dir, segment);
+    return lstatSync(fullPath).isDirectory() && recursiveDirTest(fullPath, test);
+  });
+}

packages/compiler-cli/src/ngcc/src/transform/package_transformer.ts

@@ -5,9 +5,9 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
-import {readFileSync, writeFileSync} from 'fs';
+import {existsSync, readFileSync, writeFileSync} from 'fs';
 import {dirname, relative, resolve} from 'path';
-import {mkdir} from 'shelljs';
+import {mkdir, mv} from 'shelljs';
 import * as ts from 'typescript';
 
 import {DtsFileTransformer} from '../../../ngtsc/transform';
@@ -49,9 +49,9 @@ import {getEntryPoints} from './utils';
  * - Some formats may contain multiple "modules" in a single file.
  */
 export class PackageTransformer {
-  transform(packagePath: string, format: string): void {
+  transform(packagePath: string, format: string, targetPath: string = 'node_modules'): void {
     const sourceNodeModules = this.findNodeModulesPath(packagePath);
-    const targetNodeModules = sourceNodeModules.replace(/node_modules$/, 'node_modules_ngtsc');
+    const targetNodeModules = resolve(sourceNodeModules, '..', targetPath);
     const entryPoints = getEntryPoints(packagePath, format);
 
     entryPoints.forEach(entryPoint => {
@@ -63,6 +63,7 @@ export class PackageTransformer {
       };
 
       // Create the TS program and necessary helpers.
+      // TODO : create a custom compiler host that reads from .bak files if available.
       const host = ts.createCompilerHost(options);
       const packageProgram = ts.createProgram([entryPoint.entryFileName], options, host);
       const typeChecker = packageProgram.getTypeChecker();
@@ -195,6 +196,10 @@ export class PackageTransformer {
 
   writeFile(file: FileInfo): void {
     mkdir('-p', dirname(file.path));
+    const backPath = file.path + '.bak';
+    if (existsSync(file.path) && !existsSync(backPath)) {
+      mv(file.path, backPath);
+    }
     writeFileSync(file.path, file.contents, 'utf8');
   }
 }

packages/compiler-cli/test/ngcc/ngcc_spec.ts

@@ -12,6 +12,8 @@ import {mainNgcc} from '../../src/ngcc/src/main';
 
 import {TestSupport, isInBazel, setup} from '../test_support';
 
+const OUTPUT_PATH = 'node_modules_ngtsc';
+
 describe('ngcc behavioral tests', () => {
   if (!isInBazel()) {
     // These tests should be excluded from the non-Bazel build.
@@ -26,7 +28,7 @@ describe('ngcc behavioral tests', () => {
       const {cp, mkdir, rm, set} = require('shelljs');
 
       const tempRootDir = join(tmpdir(), 'ngcc-spec', format);
-      const outputDir = 'node_modules_ngtsc';
+      const outputDir = OUTPUT_PATH;
 
       set('-e');
       rm('-rf', tempRootDir);
@@ -44,7 +46,7 @@ describe('ngcc behavioral tests', () => {
     const commonPath = join(support.basePath, 'node_modules/@angular/common');
     const format = 'fesm2015';
 
-    expect(mainNgcc([commonPath, format])).toBe(0);
+    expect(mainNgcc([format, commonPath, OUTPUT_PATH])).toBe(0);
 
     onSpecCompleted(format);
   });
@@ -53,7 +55,7 @@ describe('ngcc behavioral tests', () => {
     const commonPath = join(support.basePath, 'node_modules/@angular/common');
     const format = 'fesm5';
 
-    expect(mainNgcc([commonPath, format])).toBe(0);
+    expect(mainNgcc([format, commonPath, OUTPUT_PATH])).toBe(0);
 
     onSpecCompleted(format);
   });
@@ -62,7 +64,7 @@ describe('ngcc behavioral tests', () => {
     const commonPath = join(support.basePath, 'node_modules/@angular/common');
     const format = 'esm2015';
 
-    expect(mainNgcc([commonPath, format])).toBe(0);
+    expect(mainNgcc([format, commonPath, OUTPUT_PATH])).toBe(0);
 
     onSpecCompleted(format);
   });
@@ -71,7 +73,7 @@ describe('ngcc behavioral tests', () => {
     const commonPath = join(support.basePath, 'node_modules/@angular/common');
     const format = 'esm5';
 
-    expect(mainNgcc([commonPath, format])).toBe(0);
+    expect(mainNgcc([format, commonPath, OUTPUT_PATH])).toBe(0);
 
     onSpecCompleted(format);
   });