test: run e2e tests on provided packages

jbedard · jbedard · commit f21bbb5599fd · 2022-08-18T15:50:56.000-07:00
The NPM packages to test must be specified via --package instead of invoking the build from within the e2e tests
diff --git a/.circleci/dynamic_config.yml b/.circleci/dynamic_config.yml
@@ -237,23 +237,23 @@ jobs:
             - run:
                 name: Execute CLI E2E Tests with NPM
                 command: |
-                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --tmpdir=/mnt/ramdisk/e2e
+                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --tmpdir=/mnt/ramdisk/e2e --package ./dist/_*.tgz
       - when:
           condition:
             equal: ['esbuild', << parameters.subset >>]
           steps:
             - run:
                 name: Execute CLI E2E Tests Subset with Esbuild
                 command: |
-                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --esbuild --tmpdir=/mnt/ramdisk/e2e --glob="{tests/basic/**,tests/build/prod-build.ts,tests/build/relative-sourcemap.ts,tests/build/styles/scss.ts,tests/build/styles/include-paths.ts,tests/commands/add/add-pwa.ts}" --ignore="tests/basic/{environment,rebuild,serve,scripts-array}.ts"
+                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --esbuild --tmpdir=/mnt/ramdisk/e2e --package ./dist/_*.tgz --glob="{tests/basic/**,tests/build/prod-build.ts,tests/build/relative-sourcemap.ts,tests/build/styles/scss.ts,tests/build/styles/include-paths.ts,tests/commands/add/add-pwa.ts}" --ignore="tests/basic/{environment,rebuild,serve,scripts-array}.ts"
       - when:
           condition:
             equal: ['yarn', << parameters.subset >>]
           steps:
             - run:
                 name: Execute CLI E2E Tests Subset with Yarn
                 command: |
-                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --yarn --tmpdir=/mnt/ramdisk/e2e --glob="{tests/basic/**,tests/update/**,tests/commands/add/**}"
+                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --yarn --tmpdir=/mnt/ramdisk/e2e --package ./dist/_*.tgz --glob="{tests/basic/**,tests/update/**,tests/commands/add/**}"
       - fail_fast
 
   test-browsers:
@@ -275,7 +275,7 @@ jobs:
         # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
         # too early without Saucelabs not being ready.
       - run: ./scripts/saucelabs/wait-for-tunnel.sh
-      - run: node ./tests/legacy-cli/run_e2e ./tests/legacy-cli/e2e/tests/misc/browsers.ts
+      - run: node ./tests/legacy-cli/run_e2e --package ./dist/_*.tgz --glob="tests/misc/browsers.ts"
       - run: ./scripts/saucelabs/stop-tunnel.sh
       - fail_fast
 
@@ -284,6 +284,10 @@ jobs:
     steps:
       - custom_attach_workspace
       - run: yarn build
+      - persist_to_workspace:
+          root: *workspace_location
+          paths:
+            - dist/_*.tgz
 
   build-bazel-e2e:
     executor: action-executor
@@ -350,12 +354,14 @@ jobs:
             # Path where Arsenal Image Mounter files are downloaded.
             # Must match path in .circleci/win-ram-disk.ps1
             - ./aim
+      # Build the npm packages for the e2e tests
+      - run: yarn build
       # Run partial e2e suite on PRs only. Release branches will run the full e2e suite.
       - run:
           name: Execute E2E Tests
           command: |
             mkdir X:/ramdisk/e2e-main
-            node tests\legacy-cli\run_e2e.js --nb-shards=$env:CIRCLE_NODE_TOTAL --shard=$env:CIRCLE_NODE_INDEX --tmpdir=X:/ramdisk/e2e-main
+            node tests\legacy-cli\run_e2e.js --nb-shards=$env:CIRCLE_NODE_TOTAL --shard=$env:CIRCLE_NODE_INDEX --tmpdir=X:/ramdisk/e2e-main --package ./dist/_*.tgz
       - fail_fast
 
 workflows:
diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel
diff --git a/package.json b/package.json
@@ -119,6 +119,7 @@
     "@types/progress": "^2.0.3",
     "@types/resolve": "^1.17.1",
     "@types/semver": "^7.3.12",
+    "@types/tar": "^6.1.2",
     "@types/text-table": "^0.2.1",
     "@types/uuid": "^8.0.0",
     "@types/yargs": "^17.0.8",
diff --git a/tarify.mjs b/tarify.mjs
@@ -0,0 +1,49 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import fs from 'fs';
+import { Parse } from 'tar';
+
+['uncaughtExceptionMonitor', 'unhandledRejection'].forEach(type => {
+  process.on(type, (...args) => console.error(`${type}: `, ...args));
+});
+
+// Inspired by https://github.com/npm/node-tar/issues/181#issuecomment-402088964
+async function extractTarFile(tarball, filePath) {
+  console.log('DETAR: ', filePath, tarball);
+
+  return new Promise((resolve, reject) => {
+    let chunks = null;
+
+    fs.createReadStream(tarball).pipe(
+      new Parse({
+        strict: true,
+        end: true,
+        filter: (p) => p === filePath,
+        onentry: (entry) => {
+          chunks = [];
+
+          entry.on('data', (chunk) => chunks.push(chunk));
+          entry.on('error', reject);
+          entry.on('finish', () => resolve(Buffer.concat(chunks)));
+        },
+      }),
+    );
+  });
+}
+
+try {
+  const r = await extractTarFile(
+    './dist/bin/packages/angular_devkit/core/npm_package.tgz',
+    'package/package.json'
+  );
+
+  console.log('DONE: ', JSON.parse(r.toString('utf8')).name);
+} catch (e) {
+  console.log("FAIL", e);
+}
diff --git a/tests/legacy-cli/e2e/setup/010-local-publish.ts b/tests/legacy-cli/e2e/setup/010-local-publish.ts
@@ -1,26 +1,30 @@
+import { join } from 'path';
 import { getGlobalVariable } from '../utils/env';
 import { globalNpm, extractNpmEnv } from '../utils/process';
 import { isPrereleaseCli } from '../utils/project';
 
 export default async function () {
   const testRegistry: string = getGlobalVariable('package-registry');
-  await globalNpm(
-    [
-      'run',
-      'admin',
-      '--',
-      'publish',
-      '--no-versionCheck',
-      '--no-branchCheck',
-      `--registry=${testRegistry}`,
-      '--tag',
-      isPrereleaseCli() ? 'next' : 'latest',
-    ],
-    {
-      ...extractNpmEnv(),
-      // Also set an auth token value for the local test registry which is required by npm 7+
-      // even though it is never actually used.
-      'NPM_CONFIG__AUTH': 'e2e-testing',
-    },
+  const packageTars: string[] = Object.values(getGlobalVariable('package-tars'));
+
+  // Publish packages specified with --package
+  await Promise.all(
+    packageTars.map((p: string) =>
+      globalNpm(
+        [
+          'publish',
+          `--registry=${testRegistry}`,
+          '--tag',
+          isPrereleaseCli() ? 'next' : 'latest',
+          join(process.cwd(), p),
+        ],
+        {
+          ...extractNpmEnv(),
+          // Also set an auth token value for the local test registry which is required by npm 7+
+          // even though it is never actually used.
+          'NPM_CONFIG__AUTH': 'e2e-testing',
+        },
+      ),
+    ),
   );
 }
diff --git a/tests/legacy-cli/e2e/utils/BUILD.bazel b/tests/legacy-cli/e2e/utils/BUILD.bazel
@@ -6,11 +6,11 @@ ts_library(
     srcs = glob(["**/*.ts"]),
     visibility = ["//visibility:public"],
     deps = [
-        "//lib",
         "//tests/legacy-cli/e2e/ng-snapshot",
         "@npm//@types/glob",
         "@npm//@types/node-fetch",
         "@npm//@types/semver",
+        "@npm//@types/tar",
         "@npm//@types/yargs-parser",
         "@npm//ansi-colors",
         "@npm//glob",
@@ -19,6 +19,7 @@ ts_library(
         "@npm//puppeteer",
         "@npm//rxjs",
         "@npm//semver",
+        "@npm//tar",
         "@npm//tree-kill",
         "@npm//verdaccio",
         "@npm//verdaccio-auth-memory",
diff --git a/tests/legacy-cli/e2e/utils/project.ts b/tests/legacy-cli/e2e/utils/project.ts
@@ -2,7 +2,6 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { prerelease, SemVer } from 'semver';
 import yargsParser from 'yargs-parser';
-import { packages } from '../../../../lib/packages';
 import { getGlobalVariable } from './env';
 import { prependToFile, readFile, replaceInFile, writeFile } from './fs';
 import { gitCommit } from './git';
@@ -95,15 +94,17 @@ export async function prepareProjectForE2e(name: string) {
 }
 
 export function useBuiltPackages(): Promise<void> {
+  const packages = getGlobalVariable('package-tars');
+
   return updateJsonFile('package.json', (json) => {
     json['dependencies'] ??= {};
     json['devDependencies'] ??= {};
 
     for (const packageName of Object.keys(packages)) {
       if (packageName in json['dependencies']) {
-        json['dependencies'][packageName] = packages[packageName].tar;
+        json['dependencies'][packageName] = packages[packageName];
       } else if (packageName in json['devDependencies']) {
-        json['devDependencies'][packageName] = packages[packageName].tar;
+        json['devDependencies'][packageName] = packages[packageName];
       }
     }
   });
@@ -220,7 +221,7 @@ export async function useCIChrome(projectDir: string = ''): Promise<void> {
   }
 }
 
-export const NgCLIVersion = new SemVer(packages['@angular/cli'].version);
+export const NgCLIVersion = new SemVer(require('../../../../package.json').version);
 
 export function isPrereleaseCli(): boolean {
   return (prerelease(NgCLIVersion)?.length ?? 0) > 0;
diff --git a/tests/legacy-cli/e2e/utils/tar.ts b/tests/legacy-cli/e2e/utils/tar.ts
@@ -0,0 +1,40 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import fs from 'fs';
+import { Parse } from 'tar';
+
+/**
+ * Extract and return the contents of a single file out of a tar file.
+ *
+ * @param tarball the tar file to extract from
+ * @param filePath the path of the file to extract
+ * @returns the Buffer of file or an error on fs/tar error or file not found
+ */
+export async function extractFile(tarball: string, filePath: string): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    let chunks: Buffer[] | null = null;
+
+    fs.createReadStream(tarball)
+      .pipe(
+        new Parse({
+          strict: true,
+          filter: (p) => p === filePath,
+          // TODO: @types/tar 'entry' does not have ReadEntry.on
+          onentry: (entry: any) => {
+            chunks = [];
+
+            entry.on('data', (chunk: any) => chunks!.push(chunk));
+            entry.on('error', reject);
+            entry.on('finish', () => resolve(Buffer.concat(chunks!)));
+          },
+        }),
+      )
+      .on('close', reject);
+  });
+}
diff --git a/tests/legacy-cli/e2e_runner.ts b/tests/legacy-cli/e2e_runner.ts
@@ -10,6 +10,8 @@ import { createNpmRegistry } from './e2e/utils/registry';
 import { launchTestProcess } from './e2e/utils/process';
 import { join } from 'path';
 import { findFreePort } from './e2e/utils/network';
+import { extractFile } from './e2e/utils/tar';
+import { realpathSync } from 'fs';
 
 Error.stackTraceLimit = Infinity;
 
@@ -34,6 +36,8 @@ Error.stackTraceLimit = Infinity;
  *   --shard          Index of this processes' shard.
  *   --tmpdir=path    Override temporary directory to use for new projects.
  *   --yarn           Use yarn as package manager.
+ *   --package=path   An npm package to be published before running tests
+ *
  * If unnamed flags are passed in, the list of tests will be filtered to include only those passed.
  */
 const argv = yargsParser(process.argv.slice(2), {
@@ -49,6 +53,7 @@ const argv = yargsParser(process.argv.slice(2), {
   ],
   string: ['devkit', 'glob', 'ignore', 'reuse', 'ng-tag', 'tmpdir', 'ng-version'],
   number: ['nb-shards', 'shard'],
+  array: ['package'],
   configuration: {
     'dot-notation': false,
     'camel-case-expansion': false,
@@ -163,10 +168,11 @@ setGlobalVariable('argv', argv);
 setGlobalVariable('ci', process.env['CI']?.toLowerCase() === 'true' || process.env['CI'] === '1');
 setGlobalVariable('package-manager', argv.yarn ? 'yarn' : 'npm');
 
-Promise.all([findFreePort(), findFreePort()])
-  .then(async ([httpPort, httpsPort]) => {
+Promise.all([findFreePort(), findFreePort(), findPackageTars()])
+  .then(async ([httpPort, httpsPort, packageTars]) => {
     setGlobalVariable('package-registry', 'http://localhost:' + httpPort);
     setGlobalVariable('package-secure-registry', 'http://localhost:' + httpsPort);
+    setGlobalVariable('package-tars', packageTars);
 
     // NPM registries for the lifetime of the test execution
     const registryProcess = await createNpmRegistry(httpPort, httpPort);
@@ -315,3 +321,23 @@ function printFooter(testName: string, type: 'setup' | 'initializer' | 'test', s
   );
   console.log('');
 }
+
+// Collect the packages passed as arguments and return as {package-name => pkg-path}
+async function findPackageTars(): Promise<{ [pkg: string]: string }> {
+  const pkgs: string[] = (getGlobalVariable('argv').package as string[]).flatMap((p) =>
+    glob.sync(p),
+  );
+
+  const pkgJsons = await Promise.all(pkgs.map((pkg) => extractFile(pkg, 'package/package.json')));
+
+  return pkgs.reduce((all, pkg, i) => {
+    const json = pkgJsons[i].toString('utf8');
+    const name = JSON.parse(json).name as string;
+    if (!name) {
+      throw new Error(`Package ${pkg} - package.json name not found`);
+    }
+
+    all[name] = realpathSync(pkg);
+    return all;
+  }, {} as { [pkg: string]: string });
+}
diff --git a/yarn.lock b/yarn.lock
@@ -2418,6 +2418,14 @@
   resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310"
   integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==
 
+"@types/tar@^6.1.2":
+  version "6.1.2"
+  resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.2.tgz#e60108a7d1b08cc91bf2faf1286cc08fdad48bbe"
+  integrity sha512-bnX3RRm70/n1WMwmevdOAeDU4YP7f5JSubgnuU+yrO+xQQjwDboJj3u2NTJI5ngCQhXihqVVAH5h5J8YpdpEvg==
+  dependencies:
+    "@types/node" "*"
+    minipass "^3.3.5"
+
 "@types/text-table@^0.2.1":
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/@types/text-table/-/text-table-0.2.2.tgz#774c90cfcfbc8b4b0ebb00fecbe861dc8b1e8e26"
@@ -7713,6 +7721,13 @@ minipass@^3.0.0, minipass@^3.1.1, minipass@^3.1.6:
   dependencies:
     yallist "^4.0.0"
 
+minipass@^3.3.5:
+  version "3.3.5"
+  resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.5.tgz#6da7e53a48db8a856eeb9153d85b230a2119e819"
+  integrity sha512-rQ/p+KfKBkeNwo04U15i+hOwoVBVmekmm/HcfTkTN2t9pbQKCMm4eN5gFeqgrrSp/kH/7BYYhTIHOxGqzbBPaA==
+  dependencies:
+    yallist "^4.0.0"
+
 minizlib@^2.1.1, minizlib@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931"
