fix(@angular/cli): direct Angular schematic dependency requests to known versions

etc/api/angular_devkit/schematics/tools/index.d.ts

@@ -134,6 +134,7 @@ export declare class NodeWorkflow extends workflow.BaseWorkflow {
 
 export interface NodeWorkflowOptions {
     dryRun?: boolean;
+    engineHostCreator?: (options: NodeWorkflowOptions) => NodeModulesEngineHost;
     force?: boolean;
     optionTransforms?: OptionTransform<object, object>[];
     packageManager?: string;

packages/angular/cli/commands/update-impl.ts

@@ -14,6 +14,7 @@ import * as semver from 'semver';
 import { PackageManager } from '../lib/config/schema';
 import { Command } from '../models/command';
 import { Arguments } from '../models/interface';
+import { SchematicEngineHost } from '../models/schematic-engine-host';
 import { colors } from '../utilities/color';
 import { runTempPackageBin } from '../utilities/install-package';
 import { writeErrorToLogFile } from '../utilities/log-file';
@@ -71,6 +72,7 @@ export class UpdateCommand extends Command<UpdateCommandSchema> {
         // Otherwise, use packages from the active workspace (migrations)
         resolvePaths: [__dirname, this.context.root],
         schemaValidation: true,
+        engineHostCreator: (options) => new SchematicEngineHost(options.resolvePaths),
       },
     );
   }
@@ -265,28 +267,6 @@ export class UpdateCommand extends Command<UpdateCommandSchema> {
 
   // tslint:disable-next-line:no-big-function
   async run(options: UpdateCommandSchema & Arguments) {
-    // Check if the @angular-devkit/schematics package can be resolved from the workspace root
-    // This works around issues with packages containing migrations that cannot directly depend on the package
-    // This check can be removed once the schematic runtime handles this situation
-    try {
-      require.resolve('@angular-devkit/schematics', { paths: [this.context.root] });
-    } catch (e) {
-      if (e.code === 'MODULE_NOT_FOUND') {
-        this.logger.fatal(
-          'The "@angular-devkit/schematics" package cannot be resolved from the workspace root directory. ' +
-            'This may be due to an unsupported node modules structure.\n' +
-            'Please remove both the "node_modules" directory and the package lock file; and then reinstall.\n' +
-            'If this does not correct the problem, ' +
-            'please temporarily install the "@angular-devkit/schematics" package within the workspace. ' +
-            'It can be removed once the update is complete.',
-        );
-
-        return 1;
-      }
-
-      throw e;
-    }
-
     // Check if the current installed CLI version is older than the latest version.
     if (!disableVersionCheck && await this.checkCLILatestVersion(options.verbose, options.next)) {
       this.logger.warn(

packages/angular/cli/models/schematic-command.ts

@@ -23,7 +23,6 @@ import {
   FileSystemCollection,
   FileSystemEngine,
   FileSystemSchematic,
-  FileSystemSchematicDescription,
   NodeWorkflow,
 } from '@angular-devkit/schematics/tools';
 import * as inquirer from 'inquirer';
@@ -37,6 +36,7 @@ import { isPackageNameSafeForAnalytics } from './analytics';
 import { BaseCommandOptions, Command } from './command';
 import { Arguments, CommandContext, CommandDescription, Option } from './interface';
 import { parseArguments, parseFreeFormArguments } from './parser';
+import { SchematicEngineHost } from './schematic-engine-host';
 
 export interface BaseSchematicSchema {
   debug?: boolean;
@@ -258,6 +258,7 @@ export abstract class SchematicCommand<
           ...current,
         }),
       ],
+      engineHostCreator: (options) => new SchematicEngineHost(options.resolvePaths),
     });
 
     const getProjectName = () => {

packages/angular/cli/models/schematic-engine-host.ts

@@ -0,0 +1,189 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { RuleFactory, SchematicsException, Tree } from '@angular-devkit/schematics';
+import { NodeModulesEngineHost } from '@angular-devkit/schematics/tools';
+import { readFileSync } from 'fs';
+import { parse as parseJson } from 'jsonc-parser';
+import { dirname, resolve } from 'path';
+import { Script } from 'vm';
+
+/**
+ * Environment variable to control schematic package redirection
+ * Default: Angular schematics only
+ */
+const schematicRedirectVariable = process.env['NG_SCHEMATIC_REDIRECT']?.toLowerCase();
+
+function shouldWrapSchematic(schematicFile: string): boolean {
+  // Check environment variable if present
+  if (schematicRedirectVariable !== undefined) {
+    switch (schematicRedirectVariable) {
+      case '0':
+      case 'false':
+      case 'off':
+      case 'none':
+        return false;
+      case 'all':
+        return true;
+    }
+  }
+
+  // Never wrap `@schematics/update` when executed directly
+  // It communicates with the update command via `global`
+  if (/[\/\\]node_modules[\/\\]@schematics[\/\\]update[\/\\]/.test(schematicFile)) {
+    return false;
+  }
+
+  // Default is only first-party Angular schematic packages
+  // Angular schematics are safe to use in the wrapped VM context
+  return /[\/\\]node_modules[\/\\]@(?:angular|schematics|nguniversal)[\/\\]/.test(schematicFile);
+}
+
+export class SchematicEngineHost extends NodeModulesEngineHost {
+  protected _resolveReferenceString(refString: string, parentPath: string) {
+    const [path, name] = refString.split('#', 2);
+    // Mimic behavior of ExportStringRef class used in default behavior
+    const fullPath = path[0] === '.' ? resolve(parentPath ?? process.cwd(), path) : path;
+
+    const schematicFile = require.resolve(fullPath, { paths: [parentPath] });
+
+    if (shouldWrapSchematic(schematicFile)) {
+      const schematicPath = dirname(schematicFile);
+
+      const moduleCache = new Map<string, unknown>();
+      const factoryInitializer = wrap(
+        schematicFile,
+        schematicPath,
+        moduleCache,
+        name || 'default',
+      ) as () => RuleFactory<{}>;
+
+      const factory = factoryInitializer();
+      if (!factory || typeof factory !== 'function') {
+        return null;
+      }
+
+      return { ref: factory, path: schematicPath };
+    }
+
+    // All other schematics use default behavior
+    return super._resolveReferenceString(refString, parentPath);
+  }
+}
+
+/**
+ * Minimal shim modules for legacy deep imports of `@schematics/angular`
+ */
+const legacyModules: Record<string, unknown> = {
+  '@schematics/angular/utility/config': {
+    getWorkspace(host: Tree) {
+      const path = '/.angular.json';
+      const data = host.read(path);
+      if (!data) {
+        throw new SchematicsException(`Could not find (${path})`);
+      }
+
+      return parseJson(data.toString(), [], { allowTrailingComma: true });
+    },
+  },
+  '@schematics/angular/utility/project': {
+    buildDefaultPath(project: { sourceRoot?: string; root: string; projectType: string }): string {
+      const root = project.sourceRoot ? `/${project.sourceRoot}/` : `/${project.root}/src/`;
+
+      return `${root}${project.projectType === 'application' ? 'app' : 'lib'}`;
+    },
+  },
+};
+
+/**
+ * Wrap a JavaScript file in a VM context to allow specific Angular dependencies to be redirected.
+ * This VM setup is ONLY intended to redirect dependencies.
+ *
+ * @param schematicFile A JavaScript schematic file path that should be wrapped.
+ * @param schematicDirectory A directory that will be used as the location of the JavaScript file.
+ * @param moduleCache A map to use for caching repeat module usage and proper `instanceof` support.
+ * @param exportName An optional name of a specific export to return. Otherwise, return all exports.
+ */
+function wrap(
+  schematicFile: string,
+  schematicDirectory: string,
+  moduleCache: Map<string, unknown>,
+  exportName?: string,
+): () => unknown {
+  const { createRequire, createRequireFromPath } = require('module');
+  // Node.js 10.x does not support `createRequire` so fallback to `createRequireFromPath`
+  // `createRequireFromPath` is deprecated in 12+ and can be removed once 10.x support is removed
+  const scopedRequire = createRequire?.(schematicFile) || createRequireFromPath(schematicFile);
+
+  const customRequire = function (id: string) {
+    if (legacyModules[id]) {
+      // Provide compatibility modules for older versions of @angular/cdk
+      return legacyModules[id];
+    } else if (id.startsWith('@angular-devkit/') || id.startsWith('@schematics/')) {
+      // Resolve from inside the `@angular/cli` project
+      const packagePath = require.resolve(id);
+
+      return require(packagePath);
+    } else if (id.startsWith('.') || id.startsWith('@angular/cdk')) {
+      // Wrap relative files inside the schematic collection
+      // Also wrap `@angular/cdk`, it contains helper utilities that import core schematic packages
+
+      // Resolve from the original file
+      const modulePath = scopedRequire.resolve(id);
+
+      // Use cached module if available
+      const cachedModule = moduleCache.get(modulePath);
+      if (cachedModule) {
+        return cachedModule;
+      }
+
+      // Do not wrap vendored third-party packages or JSON files
+      if (
+        !/[\/\\]node_modules[\/\\]@schematics[\/\\]angular[\/\\]third_party[\/\\]/.test(
+          modulePath,
+        ) &&
+        !modulePath.endsWith('.json')
+      ) {
+        // Wrap module and save in cache
+        const wrappedModule = wrap(modulePath, dirname(modulePath), moduleCache)();
+        moduleCache.set(modulePath, wrappedModule);
+
+        return wrappedModule;
+      }
+    }
+
+    // All others are required directly from the original file
+    return scopedRequire(id);
+  };
+
+  // Setup a wrapper function to capture the module's exports
+  const schematicCode = readFileSync(schematicFile, 'utf8');
+  // `module` is required due to @angular/localize ng-add being in UMD format
+  const headerCode = '(function() {\nvar exports = {};\nvar module = { exports };\n';
+  const footerCode = exportName ? `\nreturn exports['${exportName}'];});` : '\nreturn exports;});';
+
+  const script = new Script(headerCode + schematicCode + footerCode, {
+    filename: schematicFile,
+    lineOffset: 3,
+  });
+
+  const context = {
+    __dirname: schematicDirectory,
+    __filename: schematicFile,
+    Buffer,
+    console,
+    process,
+    get global() {
+      return this;
+    },
+    require: customRequire,
+  };
+
+  const exportsFactory = script.runInNewContext(context);
+
+  return exportsFactory;
+}

packages/angular_devkit/schematics/tools/workflow/node-workflow.ts

@@ -25,6 +25,7 @@ export interface NodeWorkflowOptions {
   resolvePaths?: string[];
   schemaValidation?: boolean;
   optionTransforms?: OptionTransform<object, object>[];
+  engineHostCreator?: (options: NodeWorkflowOptions) => NodeModulesEngineHost;
 }
 
 /**
@@ -46,7 +47,8 @@ export class NodeWorkflow extends workflow.BaseWorkflow {
       root = options.root;
     }
 
-    const engineHost = new NodeModulesEngineHost(options.resolvePaths);
+    const engineHost =
+      options.engineHostCreator?.(options) || new NodeModulesEngineHost(options.resolvePaths);
     super({
       host,
       engineHost,

tests/legacy-cli/e2e/tests/misc/invalid-schematic-dependencies.ts

@@ -0,0 +1,21 @@
+import { join } from 'path';
+import { expectFileToMatch } from '../../utils/fs';
+import { ng } from '../../utils/process';
+import { installPackage, uninstallPackage } from '../../utils/packages';
+import { isPrereleaseCli } from '../../utils/project';
+
+export default async function () {
+  const componentDir = join('src', 'app', 'test-component');
+
+  // Install old and incompatible version
+  // Must directly use npm registry since old versions are not hosted locally
+  await installPackage('@schematics/angular@7', 'https://registry.npmjs.org')
+
+  const tag = await isPrereleaseCli() ?  '@next' : '';
+  await ng('add', `@angular/material${tag}`);
+  await expectFileToMatch('package.json', /@angular\/material/);
+
+  // Clean up existing cdk package
+  // Not doing so can cause adding material to fail if an incompatible cdk is present
+  await uninstallPackage('@angular/cdk');
+}

tests/legacy-cli/e2e/utils/packages.ts

@@ -27,12 +27,13 @@ export async function installWorkspacePackages(updateWebdriver = true): Promise<
   }
 }
 
-export async function installPackage(specifier: string): Promise<ProcessOutput> {
+export async function installPackage(specifier: string, registry?: string): Promise<ProcessOutput> {
+  const registryOption = registry ? [`--registry=${registry}`] : [];
   switch (getActivePackageManager()) {
     case 'npm':
-      return silentNpm('install', specifier);
+      return silentNpm('install', specifier, ...registryOption);
     case 'yarn':
-      return silentYarn('add', specifier);
+      return silentYarn('add', specifier, ...registryOption);
   }
 }