test: run e2e tests on pre-compiled packages

The NPM packages being tested must be pre-compiled and the tar packages specified via --package. This way the real packages such as snapshots, release artifacts or cached packages can be tested. Previously the e2e tests compiled and packaged during test execution.

Author: jbedard

.circleci/dynamic_config.yml

@@ -275,7 +275,7 @@ jobs:
         # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
         # too early without Saucelabs not being ready.
       - run: ./scripts/saucelabs/wait-for-tunnel.sh
-      - run: node ./tests/legacy-cli/run_e2e ./tests/legacy-cli/e2e/tests/misc/browsers.ts
+      - run: node ./tests/legacy-cli/run_e2e --glob="tests/misc/browsers.ts"
       - run: ./scripts/saucelabs/stop-tunnel.sh
       - fail_fast
 
@@ -284,6 +284,10 @@ jobs:
     steps:
       - custom_attach_workspace
       - run: yarn build
+      - persist_to_workspace:
+          root: *workspace_location
+          paths:
+            - dist/_*.tgz
 
   build-bazel-e2e:
     executor: action-executor
@@ -387,6 +391,8 @@ jobs:
             # Path where Arsenal Image Mounter files are downloaded.
             # Must match path in .circleci/win-ram-disk.ps1
             - ./aim
+      # Build the npm packages for the e2e tests
+      - run: yarn build
       # Run partial e2e suite on PRs only. Release branches will run the full e2e suite.
       - run:
           name: Execute E2E Tests

docs/DEVELOPER.md

@@ -82,8 +82,10 @@ You can find more info about debugging [tests with Bazel in the docs.](https://g
 
 ### End to end tests
 
-- Run: `node tests/legacy-cli/run_e2e.js`
+- Compile the packages being tested: `yarn build`
+- Run all tests: `node tests/legacy-cli/run_e2e.js`
 - Run a subset of the tests: `node tests/legacy-cli/run_e2e.js tests/legacy-cli/e2e/tests/i18n/ivy-localize-*`
+- Run on a custom set of npm packages (tar files): `node tests/legacy-cli/run_e2e.js --package _angular_cli.tgz _angular_create.tgz dist/*.tgz ...`
 
 When running the debug commands, Node will stop and wait for a debugger to attach.
 You can attach your IDE to the debugger to stop on breakpoints and step through the code. Also, see [IDE Specific Usage](#ide-specific-usage) for a

lib/BUILD.bazel

@@ -118,6 +118,7 @@
     "@types/progress": "^2.0.3",
     "@types/resolve": "^1.17.1",
     "@types/semver": "^7.3.12",
+    "@types/tar": "^6.1.2",
     "@types/text-table": "^0.2.1",
     "@types/uuid": "^8.0.0",
     "@types/yargs": "^17.0.8",

package.json

@@ -1,26 +1,30 @@
 import { getGlobalVariable } from '../utils/env';
+import { PkgInfo } from '../utils/packages';
 import { globalNpm, extractNpmEnv } from '../utils/process';
 import { isPrereleaseCli } from '../utils/project';
 
 export default async function () {
   const testRegistry: string = getGlobalVariable('package-registry');
-  await globalNpm(
-    [
-      'run',
-      'admin',
-      '--',
-      'publish',
-      '--no-versionCheck',
-      '--no-branchCheck',
-      `--registry=${testRegistry}`,
-      '--tag',
-      isPrereleaseCli() ? 'next' : 'latest',
-    ],
-    {
-      ...extractNpmEnv(),
-      // Also set an auth token value for the local test registry which is required by npm 7+
-      // even though it is never actually used.
-      'NPM_CONFIG__AUTH': 'e2e-testing',
-    },
+  const packageTars: PkgInfo[] = Object.values(getGlobalVariable('package-tars'));
+
+  // Publish packages specified with --package
+  await Promise.all(
+    packageTars.map(({ path: p }) =>
+      globalNpm(
+        [
+          'publish',
+          `--registry=${testRegistry}`,
+          '--tag',
+          isPrereleaseCli() ? 'next' : 'latest',
+          p,
+        ],
+        {
+          ...extractNpmEnv(),
+          // Also set an auth token value for the local test registry which is required by npm 7+
+          // even though it is never actually used.
+          'NPM_CONFIG__AUTH': 'e2e-testing',
+        },
+      ),
+    ),
   );
 }

tests/legacy-cli/e2e/setup/010-local-publish.ts

@@ -4,7 +4,7 @@ import { createProjectFromAsset } from '../../utils/assets';
 import { expectFileMatchToExist, readFile } from '../../utils/fs';
 import { getActivePackageManager } from '../../utils/packages';
 import { ng, noSilentNg } from '../../utils/process';
-import { isPrereleaseCli, useCIChrome, useCIDefaults, NgCLIVersion } from '../../utils/project';
+import { isPrereleaseCli, useCIChrome, useCIDefaults, getNgCLIVersion } from '../../utils/project';
 
 export default async function () {
   let restoreRegistry: (() => Promise<void>) | undefined;
@@ -32,7 +32,7 @@ export default async function () {
     const cliMajorProjectVersion = new SemVer(cliVersion).major;
 
     // CLI current version.
-    const cliMajorVersion = NgCLIVersion.major;
+    const cliMajorVersion = getNgCLIVersion().major;
 
     for (let version = cliMajorProjectVersion + 1; version < cliMajorVersion; version++) {
       // Run all the migrations until the current build major version - 1.

tests/legacy-cli/e2e/tests/update/update.ts

@@ -6,11 +6,11 @@ ts_library(
     srcs = glob(["**/*.ts"]),
     visibility = ["//visibility:public"],
     deps = [
-        "//lib",
         "//tests/legacy-cli/e2e/ng-snapshot",
         "@npm//@types/glob",
         "@npm//@types/node-fetch",
         "@npm//@types/semver",
+        "@npm//@types/tar",
         "@npm//@types/yargs-parser",
         "@npm//ansi-colors",
         "@npm//glob",
@@ -19,6 +19,7 @@ ts_library(
         "@npm//puppeteer",
         "@npm//rxjs",
         "@npm//semver",
+        "@npm//tar",
         "@npm//tree-kill",
         "@npm//verdaccio",
         "@npm//verdaccio-auth-memory",

tests/legacy-cli/e2e/utils/BUILD.bazel

@@ -1,6 +1,12 @@
 import { getGlobalVariable } from './env';
 import { ProcessOutput, silentNpm, silentYarn } from './process';
 
+export interface PkgInfo {
+  readonly name: string;
+  readonly version: string;
+  readonly path: string;
+}
+
 export function getActivePackageManager(): 'npm' | 'yarn' {
   const value = getGlobalVariable('package-manager');
   if (value && value !== 'npm' && value !== 'yarn') {

tests/legacy-cli/e2e/utils/packages.ts

@@ -2,12 +2,11 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { prerelease, SemVer } from 'semver';
 import yargsParser from 'yargs-parser';
-import { packages } from '../../../../lib/packages';
 import { getGlobalVariable } from './env';
 import { prependToFile, readFile, replaceInFile, writeFile } from './fs';
 import { gitCommit } from './git';
 import { findFreePort } from './network';
-import { installWorkspacePackages } from './packages';
+import { installWorkspacePackages, PkgInfo } from './packages';
 import { exec, execAndWaitForOutputToMatch, git, ng } from './process';
 
 export function updateJsonFile(filePath: string, fn: (json: any) => any | void) {
@@ -95,6 +94,8 @@ export async function prepareProjectForE2e(name: string) {
 }
 
 export function useBuiltPackagesVersions(): Promise<void> {
+  const packages: { [name: string]: PkgInfo } = getGlobalVariable('package-tars');
+
   return updateJsonFile('package.json', (json) => {
     json['dependencies'] ??= {};
     json['devDependencies'] ??= {};
@@ -220,8 +221,12 @@ export async function useCIChrome(projectDir: string = ''): Promise<void> {
   }
 }
 
-export const NgCLIVersion = new SemVer(packages['@angular/cli'].version);
+export function getNgCLIVersion(): SemVer {
+  const packages: { [name: string]: PkgInfo } = getGlobalVariable('package-tars');
+
+  return new SemVer(packages['@angular/cli'].version);
+}
 
 export function isPrereleaseCli(): boolean {
-  return (prerelease(NgCLIVersion)?.length ?? 0) > 0;
+  return (prerelease(getNgCLIVersion())?.length ?? 0) > 0;
 }

tests/legacy-cli/e2e/utils/project.ts

@@ -0,0 +1,39 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import fs from 'fs';
+import { normalize } from 'path';
+import { Parse } from 'tar';
+
+/**
+ * Extract and return the contents of a single file out of a tar file.
+ *
+ * @param tarball the tar file to extract from
+ * @param filePath the path of the file to extract
+ * @returns the Buffer of file or an error on fs/tar error or file not found
+ */
+export async function extractFile(tarball: string, filePath: string): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    fs.createReadStream(tarball)
+      .pipe(
+        new Parse({
+          strict: true,
+          filter: (p) => normalize(p) === normalize(filePath),
+          // TODO: @types/tar 'entry' does not have ReadEntry.on
+          onentry: (entry: any) => {
+            const chunks: Buffer[] = [];
+
+            entry.on('data', (chunk: any) => chunks!.push(chunk));
+            entry.on('error', reject);
+            entry.on('finish', () => resolve(Buffer.concat(chunks!)));
+          },
+        }),
+      )
+      .on('close', () => reject(`${tarball} does not contain ${filePath}`));
+  });
+}

tests/legacy-cli/e2e/utils/tar.ts

@@ -10,6 +10,9 @@ import { createNpmRegistry } from './e2e/utils/registry';
 import { launchTestProcess } from './e2e/utils/process';
 import { join } from 'path';
 import { findFreePort } from './e2e/utils/network';
+import { extractFile } from './e2e/utils/tar';
+import { realpathSync } from 'fs';
+import { PkgInfo } from './e2e/utils/packages';
 
 Error.stackTraceLimit = Infinity;
 
@@ -34,6 +37,8 @@ Error.stackTraceLimit = Infinity;
  *   --shard          Index of this processes' shard.
  *   --tmpdir=path    Override temporary directory to use for new projects.
  *   --yarn           Use yarn as package manager.
+ *   --package=path   An npm package to be published before running tests
+ *
  * If unnamed flags are passed in, the list of tests will be filtered to include only those passed.
  */
 const argv = yargsParser(process.argv.slice(2), {
@@ -49,10 +54,14 @@ const argv = yargsParser(process.argv.slice(2), {
   ],
   string: ['devkit', 'glob', 'ignore', 'reuse', 'ng-tag', 'tmpdir', 'ng-version'],
   number: ['nb-shards', 'shard'],
+  array: ['package'],
   configuration: {
     'dot-notation': false,
     'camel-case-expansion': false,
   },
+  default: {
+    'package': ['./dist/_*.tgz'],
+  },
 });
 
 /**
@@ -162,10 +171,11 @@ console.log(['Tests:', ...testsToRun].join('\n '));
 setGlobalVariable('argv', argv);
 setGlobalVariable('package-manager', argv.yarn ? 'yarn' : 'npm');
 
-Promise.all([findFreePort(), findFreePort()])
-  .then(async ([httpPort, httpsPort]) => {
+Promise.all([findFreePort(), findFreePort(), findPackageTars()])
+  .then(async ([httpPort, httpsPort, packageTars]) => {
     setGlobalVariable('package-registry', 'http://localhost:' + httpPort);
     setGlobalVariable('package-secure-registry', 'http://localhost:' + httpsPort);
+    setGlobalVariable('package-tars', packageTars);
 
     // NPM registries for the lifetime of the test execution
     const registryProcess = await createNpmRegistry(httpPort, httpPort);
@@ -306,3 +316,23 @@ function printFooter(testName: string, type: 'setup' | 'initializer' | 'test', s
   );
   console.log('');
 }
+
+// Collect the packages passed as arguments and return as {package-name => pkg-path}
+async function findPackageTars(): Promise<{ [pkg: string]: PkgInfo }> {
+  const pkgs: string[] = (getGlobalVariable('argv').package as string[]).flatMap((p) =>
+    glob.sync(p, { realpath: true }),
+  );
+
+  const pkgJsons = await Promise.all(pkgs.map((pkg) => extractFile(pkg, './package/package.json')));
+
+  return pkgs.reduce((all, pkg, i) => {
+    const json = pkgJsons[i].toString('utf8');
+    const { name, version } = JSON.parse(json);
+    if (!name) {
+      throw new Error(`Package ${pkg} - package.json name/version not found`);
+    }
+
+    all[name] = { path: realpathSync(pkg), name, version };
+    return all;
+  }, {} as { [pkg: string]: PkgInfo });
+}

tests/legacy-cli/e2e_runner.ts

@@ -2538,6 +2538,14 @@
   resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310"
   integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==
 
+"@types/tar@^6.1.2":
+  version "6.1.2"
+  resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.2.tgz#e60108a7d1b08cc91bf2faf1286cc08fdad48bbe"
+  integrity sha512-bnX3RRm70/n1WMwmevdOAeDU4YP7f5JSubgnuU+yrO+xQQjwDboJj3u2NTJI5ngCQhXihqVVAH5h5J8YpdpEvg==
+  dependencies:
+    "@types/node" "*"
+    minipass "^3.3.5"
+
 "@types/text-table@^0.2.1":
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/@types/text-table/-/text-table-0.2.2.tgz#774c90cfcfbc8b4b0ebb00fecbe861dc8b1e8e26"
@@ -7710,6 +7718,13 @@ minipass@^3.0.0, minipass@^3.1.1, minipass@^3.1.6:
   dependencies:
     yallist "^4.0.0"
 
+minipass@^3.3.5:
+  version "3.3.5"
+  resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.5.tgz#6da7e53a48db8a856eeb9153d85b230a2119e819"
+  integrity sha512-rQ/p+KfKBkeNwo04U15i+hOwoVBVmekmm/HcfTkTN2t9pbQKCMm4eN5gFeqgrrSp/kH/7BYYhTIHOxGqzbBPaA==
+  dependencies:
+    yallist "^4.0.0"
+
 minizlib@^2.1.1, minizlib@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931"