test: run tests in isolated subprocess

Author: jbedard

tests/legacy-cli/e2e/setup/300-log-environment.ts renamed to tests/legacy-cli/e2e/initialize/300-log-environment.ts

@@ -2,7 +2,7 @@ import { join } from 'path';
 import { getGlobalVariable } from '../utils/env';
 import { expectFileToExist } from '../utils/fs';
 import { gitClean } from '../utils/git';
-import { setRegistry as setNPMConfigRegistry } from '../utils/packages';
+import { installPackage, setRegistry as setNPMConfigRegistry } from '../utils/packages';
 import { ng } from '../utils/process';
 import { prepareProjectForE2e, updateJsonFile } from '../utils/project';
 
@@ -22,6 +22,10 @@ export default async function () {
     // Ensure local test registry is used when outside a project
     await setNPMConfigRegistry(true);
 
+    // Install puppeteer in the parent directory of the test projects for use
+    // by the CLI within any test project
+    await installPackage('[email protected]');
+
     await ng('new', 'test-project', '--skip-install', ...extraArgs);
     await expectFileToExist(join(process.cwd(), 'test-project'));
     process.chdir('./test-project');

tests/legacy-cli/e2e/setup/500-create-project.ts renamed to tests/legacy-cli/e2e/initialize/500-create-project.ts

@@ -1,31 +1,26 @@
 import { mkdir, writeFile } from 'fs/promises';
-import { delimiter, join } from 'path';
-import { getGlobalVariable } from '../utils/env';
+import { join } from 'path';
+import { getGlobalVariable, setGlobalVariable } from '../utils/env';
 
 /**
  * Configure npm to use a unique sandboxed environment.
  */
 export default async function () {
   const tempRoot: string = getGlobalVariable('tmp-root');
   const npmModulesPrefix = join(tempRoot, 'npm-global');
+  const npmRegistry: string = getGlobalVariable('package-registry');
   const npmrc = join(tempRoot, '.npmrc');
 
   // Configure npm to use the sandboxed npm globals and rc file
+  // From this point onward all npm transactions use the "global" npm cache
+  // isolated within this e2e test invocation.
   process.env.NPM_CONFIG_USERCONFIG = npmrc;
   process.env.NPM_CONFIG_PREFIX = npmModulesPrefix;
 
-  // Ensure the custom npm global bin is first on the PATH
-  // https://docs.npmjs.com/cli/v8/configuring-npm/folders#executables
-  if (process.platform.startsWith('win')) {
-    process.env.PATH = npmModulesPrefix + delimiter + process.env.PATH;
-  } else {
-    process.env.PATH = join(npmModulesPrefix, 'bin') + delimiter + process.env.PATH;
-  }
-
-  // Ensure the globals directory and npmrc file exist.
-  // Configure the registry in the npmrc in addition to the environment variable.
-  await writeFile(npmrc, 'registry=' + getGlobalVariable('package-registry'));
+  // Configure the registry and prefix used within the test sandbox
+  await writeFile(npmrc, `registry=${npmRegistry}\nprefix=${npmModulesPrefix}`);
   await mkdir(npmModulesPrefix);
 
+  setGlobalVariable('npm-root', npmModulesPrefix);
   console.log(`  Using "${npmModulesPrefix}" as e2e test global npm cache.`);
 }

tests/legacy-cli/e2e/setup/002-npm-sandbox.ts

@@ -1,5 +1,8 @@
 import { getGlobalVariable } from '../utils/env';
-import { exec, silentNpm } from '../utils/process';
+import { silentNpm } from '../utils/process';
+
+const NPM_VERSION = '7.5.6';
+const YARN_VERSION = '1.22.18';
 
 export default async function () {
   const argv = getGlobalVariable('argv');
@@ -9,10 +12,13 @@ export default async function () {
 
   const testRegistry = getGlobalVariable('package-registry');
 
-  // Install global Angular CLI.
-  await silentNpm('install', '--global', '@angular/cli', `--registry=${testRegistry}`);
-
-  try {
-    await exec(process.platform.startsWith('win') ? 'where' : 'which', 'ng');
-  } catch {}
+  // Install global Angular CLI being tested, npm+yarn used by e2e tests.
+  await silentNpm(
+    'install',
+    '--global',
+    `--registry=${testRegistry}`,
+    '@angular/cli',
+    `npm@${NPM_VERSION}`,
+    `yarn@${YARN_VERSION}`,
+  );
 }

tests/legacy-cli/e2e/setup/100-global-cli.ts

@@ -59,13 +59,6 @@ export default function () {
       // Should run side-by-side with `ng serve`
       .then(() => execAndWaitForOutputToMatch('ng', ['serve'], / Compiled successfully./))
       .then(() => ng('e2e', 'test-project', '--dev-server-target='))
-      // Should fail without updated webdriver
-      .then(() => replaceInFile('e2e/protractor.conf.js', /chromeDriver: String.raw`[^`]*`,/, ''))
-      .then(() =>
-        expectToFail(() =>
-          ng('e2e', 'test-project', '--no-webdriver-update', '--dev-server-target='),
-        ),
-      )
       .then(
         () => killAllProcesses(),
         (err) => {

tests/legacy-cli/e2e/tests/basic/e2e.ts

@@ -3,7 +3,7 @@ import { createProjectFromAsset } from '../../../utils/assets';
 import { expectFileSizeToBeUnder, expectFileToMatch, replaceInFile } from '../../../utils/fs';
 import { execWithEnv } from '../../../utils/process';
 
-export default async function (skipCleaning: () => void) {
+export default async function () {
   const webpackCLIBin = normalize('node_modules/.bin/webpack-cli');
 
   await createProjectFromAsset('webpack/test-app');
@@ -30,6 +30,4 @@ export default async function (skipCleaning: () => void) {
     'DISABLE_V8_COMPILE_CACHE': '1',
   });
   await expectFileToMatch('dist/app.main.js', 'AppModule');
-
-  skipCleaning();
 }

tests/legacy-cli/e2e/tests/packages/webpack/test-app.ts

@@ -1,12 +1,22 @@
-const global: { [name: string]: any } = Object.create(null);
+export const ENV_PREFIX = 'LEGACI_CLI__';
 
 export function setGlobalVariable(name: string, value: any) {
-  global[name] = value;
+  process.env[ENV_PREFIX + name] = JSON.stringify(value);
 }
 
 export function getGlobalVariable(name: string): any {
-  if (!(name in global)) {
+  const envName = ENV_PREFIX + name;
+  if (!(envName in process.env)) {
     throw new Error(`Trying to access variable "${name}" but it's not defined.`);
   }
-  return global[name];
+  return JSON.parse(process.env[envName]);
+}
+
+export function getGlobalVariablesEnv(): { [k: string]: string } {
+  return Object.keys(process.env)
+    .filter((v) => v.startsWith(ENV_PREFIX))
+    .reduce((vars, n) => {
+      vars[n] = process.env[n];
+      return vars;
+    }, {});
 }

tests/legacy-cli/e2e/utils/env.ts

@@ -3,12 +3,14 @@ import { SpawnOptions } from 'child_process';
 import * as child_process from 'child_process';
 import { concat, defer, EMPTY, from } from 'rxjs';
 import { repeat, takeLast } from 'rxjs/operators';
-import { getGlobalVariable } from './env';
+import { getGlobalVariable, getGlobalVariablesEnv } from './env';
 import { catchError } from 'rxjs/operators';
+import { delimiter, join, resolve } from 'path';
 const treeKill = require('tree-kill');
 
 interface ExecOptions {
   silent?: boolean;
+  announce?: boolean;
   waitForMatch?: RegExp;
   env?: { [varname: string]: string };
   stdin?: string;
@@ -24,18 +26,22 @@ export type ProcessOutput = {
   stderr: string;
 };
 
-function _exec(options: ExecOptions, cmd: string, args: string[]): Promise<ProcessOutput> {
+async function _exec(options: ExecOptions, cmd: string, args: string[]): Promise<ProcessOutput> {
   // Create a separate instance to prevent unintended global changes to the color configuration
   // Create function is not defined in the typings. See: https://github.com/doowb/ansi-colors/pull/44
   const colors = (ansiColors as typeof ansiColors & { create: () => typeof ansiColors }).create();
 
+  const announce = options.announce ?? true;
+
   let stdout = '';
   let stderr = '';
   const cwd = options.cwd ?? process.cwd();
   const env = options.env;
-  console.log(
-    `==========================================================================================`,
-  );
+  if (announce) {
+    console.log(
+      `==========================================================================================`,
+    );
+  }
 
   args = args.filter((x) => x !== undefined);
   const flags = [
@@ -46,9 +52,15 @@ function _exec(options: ExecOptions, cmd: string, args: string[]): Promise<Proce
     .join(', ')
     .replace(/^(.+)$/, ' [$1]'); // Proper formatting.
 
-  console.log(colors.blue(`Running \`${cmd} ${args.map((x) => `"${x}"`).join(' ')}\`${flags}...`));
-  console.log(colors.blue(`CWD: ${cwd}`));
-  console.log(colors.blue(`ENV: ${JSON.stringify(env)}`));
+  if (announce) {
+    console.log(
+      colors.blue(`Running \`${cmd} ${args.map((x) => `"${x}"`).join(' ')}\`${flags}...`),
+    );
+    console.log(colors.blue(`CWD: ${cwd}`));
+
+    console.log(colors.blue(`ENV: ${JSON.stringify(env)}`));
+  }
+
   const spawnOptions: SpawnOptions = {
     cwd,
     ...(env ? { env } : {}),
@@ -140,7 +152,7 @@ function _exec(options: ExecOptions, cmd: string, args: string[]): Promise<Proce
   });
 }
 
-export function extractNpmEnv() {
+export function extractNpmEnv(): { [k: string]: string } {
   return Object.keys(process.env)
     .filter((v) => NPM_CONFIG_RE.test(v))
     .reduce(
@@ -154,6 +166,16 @@ export function extractNpmEnv() {
     );
 }
 
+function getNpmSandboxEnv(): { [k: string]: string } {
+  const tempRoot: string = getGlobalVariable('tmp-root');
+  const npmModulesPrefix: string = getGlobalVariable('npm-root');
+
+  return {
+    NPM_CONFIG_USERCONFIG: join(tempRoot, '.npmrc'),
+    NPM_CONFIG_PREFIX: npmModulesPrefix,
+  };
+}
+
 export function waitForAnyProcessOutputToMatch(
   match: RegExp,
   timeout = 30000,
@@ -299,22 +321,21 @@ export function silentNpm(
       {
         silent: true,
         cwd: (options as { cwd?: string } | undefined)?.cwd,
-        env: extractNpmEnv(),
       },
       'npm',
       params,
     );
   } else {
-    return _exec({ silent: true, env: extractNpmEnv() }, 'npm', args as string[]);
+    return _exec({ silent: true }, 'npm', args as string[]);
   }
 }
 
 export function silentYarn(...args: string[]) {
-  return _exec({ silent: true, env: extractNpmEnv() }, 'yarn', args);
+  return _exec({ silent: true }, 'yarn', args);
 }
 
 export function npm(...args: string[]) {
-  return _exec({ env: extractNpmEnv() }, 'npm', args);
+  return _exec({}, 'npm', args);
 }
 
 export function node(...args: string[]) {
@@ -328,3 +349,40 @@ export function git(...args: string[]) {
 export function silentGit(...args: string[]) {
   return _exec({ silent: true }, 'git', args);
 }
+
+/**
+ * Launch the given entry in an child process isolated to the test environment.
+ *
+ * The test environment includes the local NPM registry, isolated NPM globals,
+ * the PATH variable only referencing the local node_modules and local NPM
+ * registry (not the test runner or standard global node_modules).
+ */
+export async function launchTestProcess(entry: string, ...args: any[]) {
+  const tempRoot: string = getGlobalVariable('tmp-root');
+
+  // Extract explicit environment variables for the test process.
+  const env = {
+    ...extractNpmEnv(),
+    ...getGlobalVariablesEnv(),
+    ...getNpmSandboxEnv(),
+  };
+
+  // Modify the PATH environment variable...
+  let paths = process.env.PATH.split(delimiter);
+
+  // Only include paths within the sandboxed test environment or external
+  // non angular-cli paths such as /usr/bin for generic commands.
+  paths = paths.filter((p) => p.startsWith(tempRoot) || !p.includes('angular-cli'));
+
+  // Ensure the custom npm global bin is on the PATH
+  // https://docs.npmjs.com/cli/v8/configuring-npm/folders#executables
+  if (process.platform.startsWith('win')) {
+    paths.unshift(env.NPM_CONFIG_PREFIX);
+  } else {
+    paths.unshift(join(env.NPM_CONFIG_PREFIX, 'bin'));
+  }
+
+  env.PATH = paths.join(delimiter);
+
+  return _exec({ env }, process.execPath, [resolve(__dirname, 'run_test_process'), entry, ...args]);
+}

tests/legacy-cli/e2e/utils/process.ts

@@ -6,7 +6,7 @@ import { getGlobalVariable } from './env';
 import { prependToFile, readFile, replaceInFile, writeFile } from './fs';
 import { gitCommit } from './git';
 import { installWorkspacePackages } from './packages';
-import { execAndWaitForOutputToMatch, git, ng } from './process';
+import { exec, execAndWaitForOutputToMatch, git, ng } from './process';
 
 export function updateJsonFile(filePath: string, fn: (json: any) => any | void) {
   return readFile(filePath).then((tsConfigJson) => {
@@ -42,6 +42,26 @@ export async function prepareProjectForE2e(name) {
 
   await ng('generate', 'e2e', '--related-app-name', name);
 
+  // Initialize welenium webdrivers.
+  // Often fails the first time so attempt twice if necessary.
+  const webdriverCommand = exec.bind(
+    null,
+    'node',
+    'node_modules/protractor/bin/webdriver-manager',
+    'update',
+    '--standalone',
+    'false',
+    '--gecko',
+    'false',
+    '--versions.chrome',
+    '101.0.4951.41',
+  );
+  try {
+    await webdriverCommand();
+  } catch (e) {
+    await webdriverCommand();
+  }
+
   await useCIChrome('e2e');
   await useCIChrome('');
 

tests/legacy-cli/e2e/utils/project.ts

@@ -0,0 +1,3 @@
+'use strict';
+require('../../../../lib/bootstrap-local');
+require('./test_process');

tests/legacy-cli/e2e/utils/run_test_process.js

@@ -0,0 +1,22 @@
+import { killAllProcesses } from './process';
+
+const testScript: string = process.argv[2];
+const testModule = require(testScript);
+const testFunction: () => Promise<void> | void =
+  typeof testModule == 'function'
+    ? testModule
+    : typeof testModule.default == 'function'
+    ? testModule.default
+    : () => {
+        throw new Error('Invalid test module.');
+      };
+
+(async () => Promise.resolve(testFunction()))()
+  .finally(killAllProcesses)
+  .then(
+    () => process.exit(0),
+    (e) => {
+      console.error(e);
+      process.exit(-1);
+    },
+  );