test: run tests in isolated subprocess

Author: jbedard

tests/legacy-cli/BUILD.bazel

@@ -20,6 +20,7 @@ ts_library(
 
         # Loaded dynamically at runtime, not compiletime deps
         "//tests/legacy-cli/e2e/setup",
+        "//tests/legacy-cli/e2e/initialize",
         "//tests/legacy-cli/e2e/tests",
     ],
 )

tests/legacy-cli/e2e/setup/300-log-environment.ts renamed to tests/legacy-cli/e2e/initialize/300-log-environment.ts

@@ -1,13 +1,14 @@
 import { join } from 'path';
+import yargsParser from 'yargs-parser';
 import { getGlobalVariable } from '../utils/env';
 import { expectFileToExist } from '../utils/fs';
 import { gitClean } from '../utils/git';
-import { setRegistry as setNPMConfigRegistry } from '../utils/packages';
+import { installPackage, setRegistry as setNPMConfigRegistry } from '../utils/packages';
 import { ng } from '../utils/process';
 import { prepareProjectForE2e, updateJsonFile } from '../utils/project';
 
 export default async function () {
-  const argv = getGlobalVariable('argv');
+  const argv = getGlobalVariable<yargsParser.Arguments>('argv');
 
   if (argv.noproject) {
     return;
@@ -20,6 +21,14 @@ export default async function () {
     // Ensure local test registry is used when outside a project
     await setNPMConfigRegistry(true);
 
+    // Install puppeteer in the parent directory for use by the CLI within any test project.
+    // Align the version with the primary project package.json.
+    const puppeteerVersion = require('../../../../package.json').devDependencies.puppeteer.replace(
+      /^[\^~]/,
+      '',
+    );
+    await installPackage(`puppeteer@${puppeteerVersion}`);
+
     await ng('new', 'test-project', '--skip-install');
     await expectFileToExist(join(process.cwd(), 'test-project'));
     process.chdir('./test-project');

tests/legacy-cli/e2e/setup/500-create-project.ts renamed to tests/legacy-cli/e2e/initialize/500-create-project.ts

@@ -0,0 +1,15 @@
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "initialize",
+    testonly = True,
+    srcs = glob(["**/*.ts"]),
+    data = [
+        "//:package.json",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tests/legacy-cli/e2e/utils",
+        "@npm//@types/yargs-parser",
+    ],
+)

tests/legacy-cli/e2e/initialize/BUILD.bazel

@@ -1,30 +1,24 @@
 import { mkdir, writeFile } from 'fs/promises';
-import { delimiter, join } from 'path';
-import { getGlobalVariable } from '../utils/env';
+import { join } from 'path';
+import { getGlobalVariable, setGlobalVariable } from '../utils/env';
 
 /**
  * Configure npm to use a unique sandboxed environment.
  */
 export default async function () {
   const tempRoot: string = getGlobalVariable('tmp-root');
   const npmModulesPrefix = join(tempRoot, 'npm-global');
+  const npmRegistry: string = getGlobalVariable('package-registry');
   const npmrc = join(tempRoot, '.npmrc');
 
   // Configure npm to use the sandboxed npm globals and rc file
+  // From this point onward all npm transactions use the "global" npm cache
+  // isolated within this e2e test invocation.
   process.env.NPM_CONFIG_USERCONFIG = npmrc;
   process.env.NPM_CONFIG_PREFIX = npmModulesPrefix;
 
-  // Ensure the custom npm global bin is first on the PATH
-  // https://docs.npmjs.com/cli/v8/configuring-npm/folders#executables
-  if (process.platform.startsWith('win')) {
-    process.env.PATH = npmModulesPrefix + delimiter + process.env.PATH;
-  } else {
-    process.env.PATH = join(npmModulesPrefix, 'bin') + delimiter + process.env.PATH;
-  }
-
-  // Ensure the globals directory and npmrc file exist.
-  // Configure the registry in the npmrc in addition to the environment variable.
-  await writeFile(npmrc, 'registry=' + getGlobalVariable('package-registry'));
+  // Configure the registry and prefix used within the test sandbox
+  await writeFile(npmrc, `registry=${npmRegistry}\nprefix=${npmModulesPrefix}`);
   await mkdir(npmModulesPrefix);
 
   console.log(`  Using "${npmModulesPrefix}" as e2e test global npm cache.`);

tests/legacy-cli/e2e/setup/002-npm-sandbox.ts

@@ -1,5 +1,8 @@
 import { getGlobalVariable } from '../utils/env';
-import { exec, silentNpm } from '../utils/process';
+import { silentNpm } from '../utils/process';
+
+const NPM_VERSION = '7.24.0';
+const YARN_VERSION = '1.22.18';
 
 export default async function () {
   const argv = getGlobalVariable('argv');
@@ -9,10 +12,13 @@ export default async function () {
 
   const testRegistry: string = getGlobalVariable('package-registry');
 
-  // Install global Angular CLI.
-  await silentNpm('install', '--global', '@angular/cli', `--registry=${testRegistry}`);
-
-  try {
-    await exec(process.platform.startsWith('win') ? 'where' : 'which', 'ng');
-  } catch {}
+  // Install global Angular CLI being tested, npm+yarn used by e2e tests.
+  await silentNpm(
+    'install',
+    '--global',
+    `--registry=${testRegistry}`,
+    '@angular/cli',
+    `npm@${NPM_VERSION}`,
+    `yarn@${YARN_VERSION}`,
+  );
 }

tests/legacy-cli/e2e/setup/100-global-cli.ts

@@ -59,13 +59,6 @@ export default function () {
       // Should run side-by-side with `ng serve`
       .then(() => execAndWaitForOutputToMatch('ng', ['serve'], / Compiled successfully./))
       .then(() => ng('e2e', 'test-project', '--dev-server-target='))
-      // Should fail without updated webdriver
-      .then(() => replaceInFile('e2e/protractor.conf.js', /chromeDriver: String.raw`[^`]*`,/, ''))
-      .then(() =>
-        expectToFail(() =>
-          ng('e2e', 'test-project', '--no-webdriver-update', '--dev-server-target='),
-        ),
-      )
       .finally(() => killAllProcesses())
   );
 }

tests/legacy-cli/e2e/tests/basic/e2e.ts

@@ -3,7 +3,7 @@ import { createProjectFromAsset } from '../../../utils/assets';
 import { expectFileSizeToBeUnder, expectFileToMatch, replaceInFile } from '../../../utils/fs';
 import { execWithEnv } from '../../../utils/process';
 
-export default async function (skipCleaning: () => void) {
+export default async function () {
   const webpackCLIBin = normalize('node_modules/.bin/webpack-cli');
 
   await createProjectFromAsset('webpack/test-app');
@@ -30,6 +30,4 @@ export default async function (skipCleaning: () => void) {
     'DISABLE_V8_COMPILE_CACHE': '1',
   });
   await expectFileToMatch('dist/app.main.js', 'AppModule');
-
-  skipCleaning();
 }

tests/legacy-cli/e2e/tests/packages/webpack/test-app.ts

@@ -1,12 +1,26 @@
-const global: { [name: string]: any } = Object.create(null);
+const ENV_PREFIX = 'LEGACY_CLI__';
 
 export function setGlobalVariable(name: string, value: any) {
-  global[name] = value;
+  if (value === undefined) {
+    delete process.env[ENV_PREFIX + name];
+  } else {
+    process.env[ENV_PREFIX + name] = JSON.stringify(value);
+  }
 }
 
 export function getGlobalVariable<T = any>(name: string): T {
-  if (!(name in global)) {
+  const value = process.env[ENV_PREFIX + name];
+  if (value === undefined) {
     throw new Error(`Trying to access variable "${name}" but it's not defined.`);
   }
-  return global[name] as T;
+  return JSON.parse(value) as T;
+}
+
+export function getGlobalVariablesEnv(): NodeJS.ProcessEnv {
+  return Object.keys(process.env)
+    .filter((v) => v.startsWith(ENV_PREFIX))
+    .reduce<NodeJS.ProcessEnv>((vars, n) => {
+      vars[n] = process.env[n];
+      return vars;
+    }, {});
 }

tests/legacy-cli/e2e/utils/env.ts

@@ -3,9 +3,10 @@ import { SpawnOptions } from 'child_process';
 import * as child_process from 'child_process';
 import { concat, defer, EMPTY, from } from 'rxjs';
 import { repeat, takeLast } from 'rxjs/operators';
-import { getGlobalVariable } from './env';
+import { getGlobalVariable, getGlobalVariablesEnv } from './env';
 import { catchError } from 'rxjs/operators';
 import treeKill from 'tree-kill';
+import { delimiter, join, resolve } from 'path';
 
 interface ExecOptions {
   silent?: boolean;
@@ -300,22 +301,21 @@ export function silentNpm(
       {
         silent: true,
         cwd: (options as { cwd?: string } | undefined)?.cwd,
-        env: extractNpmEnv(),
       },
       'npm',
       params,
     );
   } else {
-    return _exec({ silent: true, env: extractNpmEnv() }, 'npm', args as string[]);
+    return _exec({ silent: true }, 'npm', args as string[]);
   }
 }
 
 export function silentYarn(...args: string[]) {
-  return _exec({ silent: true, env: extractNpmEnv() }, 'yarn', args);
+  return _exec({ silent: true }, 'yarn', args);
 }
 
 export function npm(...args: string[]) {
-  return _exec({ env: extractNpmEnv() }, 'npm', args);
+  return _exec({}, 'npm', args);
 }
 
 export function node(...args: string[]) {
@@ -329,3 +329,39 @@ export function git(...args: string[]) {
 export function silentGit(...args: string[]) {
   return _exec({ silent: true }, 'git', args);
 }
+
+/**
+ * Launch the given entry in an child process isolated to the test environment.
+ *
+ * The test environment includes the local NPM registry, isolated NPM globals,
+ * the PATH variable only referencing the local node_modules and local NPM
+ * registry (not the test runner or standard global node_modules).
+ */
+export async function launchTestProcess(entry: string, ...args: any[]) {
+  const tempRoot: string = getGlobalVariable('tmp-root');
+
+  // Extract explicit environment variables for the test process.
+  const env: NodeJS.ProcessEnv = {
+    ...extractNpmEnv(),
+    ...getGlobalVariablesEnv(),
+  };
+
+  // Modify the PATH environment variable...
+  let paths = process.env.PATH!.split(delimiter);
+
+  // Only include paths within the sandboxed test environment or external
+  // non angular-cli paths such as /usr/bin for generic commands.
+  paths = paths.filter((p) => p.startsWith(tempRoot) || !p.includes('angular-cli'));
+
+  // Ensure the custom npm global bin is on the PATH
+  // https://docs.npmjs.com/cli/v8/configuring-npm/folders#executables
+  if (process.platform.startsWith('win')) {
+    paths.unshift(env.NPM_CONFIG_PREFIX!);
+  } else {
+    paths.unshift(join(env.NPM_CONFIG_PREFIX!, 'bin'));
+  }
+
+  env.PATH = paths.join(delimiter);
+
+  return _exec({ env }, process.execPath, [resolve(__dirname, 'run_test_process'), entry, ...args]);
+}

tests/legacy-cli/e2e/utils/process.ts

@@ -7,7 +7,7 @@ import { getGlobalVariable } from './env';
 import { prependToFile, readFile, replaceInFile, writeFile } from './fs';
 import { gitCommit } from './git';
 import { installWorkspacePackages } from './packages';
-import { execAndWaitForOutputToMatch, git, ng } from './process';
+import { exec, execAndWaitForOutputToMatch, git, ng } from './process';
 
 export function updateJsonFile(filePath: string, fn: (json: any) => any | void) {
   return readFile(filePath).then((tsConfigJson) => {
@@ -42,6 +42,26 @@ export async function prepareProjectForE2e(name: string) {
 
   await ng('generate', 'e2e', '--related-app-name', name);
 
+  // Initialize selenium webdriver.
+  // Often fails the first time so attempt twice if necessary.
+  const runWebdriverUpdate = () =>
+    exec(
+      'node',
+      'node_modules/protractor/bin/webdriver-manager',
+      'update',
+      '--standalone',
+      'false',
+      '--gecko',
+      'false',
+      '--versions.chrome',
+      '101.0.4951.41',
+    );
+  try {
+    await runWebdriverUpdate();
+  } catch (e) {
+    await runWebdriverUpdate();
+  }
+
   await useCIChrome('e2e');
   await useCIChrome('');
 

tests/legacy-cli/e2e/utils/project.ts

@@ -0,0 +1,3 @@
+'use strict';
+require('../../../../lib/bootstrap-local');
+require('./test_process');

tests/legacy-cli/e2e/utils/run_test_process.js

@@ -0,0 +1,19 @@
+import { killAllProcesses } from './process';
+
+const testScript: string = process.argv[2];
+const testModule = require(testScript);
+const testFunction: () => Promise<void> | void =
+  typeof testModule == 'function'
+    ? testModule
+    : typeof testModule.default == 'function'
+    ? testModule.default
+    : () => {
+        throw new Error('Invalid test module.');
+      };
+
+(async () => Promise.resolve(testFunction()))()
+  .finally(killAllProcesses)
+  .catch((e) => {
+    console.error(e);
+    process.exitCode = -1;
+  });