test: run e2e tests on provided packages

The NPM packages to test must be specified via --package instead of invoking the build from within the e2e tests

Author: jbedard

.circleci/dynamic_config.yml

@@ -237,23 +237,23 @@ jobs:
             - run:
                 name: Execute CLI E2E Tests with NPM
                 command: |
-                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --tmpdir=/mnt/ramdisk/e2e
+                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --tmpdir=/mnt/ramdisk/e2e --package ./dist/_*.tgz
       - when:
           condition:
             equal: ['esbuild', << parameters.subset >>]
           steps:
             - run:
                 name: Execute CLI E2E Tests Subset with Esbuild
                 command: |
-                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --esbuild --tmpdir=/mnt/ramdisk/e2e --glob="{tests/basic/**,tests/build/prod-build.ts,tests/build/relative-sourcemap.ts,tests/build/styles/scss.ts,tests/build/styles/include-paths.ts,tests/commands/add/add-pwa.ts}" --ignore="tests/basic/{environment,rebuild,serve,scripts-array}.ts"
+                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --esbuild --tmpdir=/mnt/ramdisk/e2e --package ./dist/_*.tgz --glob="{tests/basic/**,tests/build/prod-build.ts,tests/build/relative-sourcemap.ts,tests/build/styles/scss.ts,tests/build/styles/include-paths.ts,tests/commands/add/add-pwa.ts}" --ignore="tests/basic/{environment,rebuild,serve,scripts-array}.ts"
       - when:
           condition:
             equal: ['yarn', << parameters.subset >>]
           steps:
             - run:
                 name: Execute CLI E2E Tests Subset with Yarn
                 command: |
-                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --yarn --tmpdir=/mnt/ramdisk/e2e --glob="{tests/basic/**,tests/update/**,tests/commands/add/**}"
+                  node ./tests/legacy-cli/run_e2e --nb-shards=${CIRCLE_NODE_TOTAL} --shard=${CIRCLE_NODE_INDEX} <<# parameters.snapshots >>--ng-snapshots<</ parameters.snapshots >> --yarn --tmpdir=/mnt/ramdisk/e2e --package ./dist/_*.tgz --glob="{tests/basic/**,tests/update/**,tests/commands/add/**}"
       - fail_fast
 
   test-browsers:
@@ -275,7 +275,7 @@ jobs:
         # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
         # too early without Saucelabs not being ready.
       - run: ./scripts/saucelabs/wait-for-tunnel.sh
-      - run: node ./tests/legacy-cli/run_e2e ./tests/legacy-cli/e2e/tests/misc/browsers.ts
+      - run: node ./tests/legacy-cli/run_e2e --package ./dist/_*.tgz --glob="tests/misc/browsers.ts"
       - run: ./scripts/saucelabs/stop-tunnel.sh
       - fail_fast
 
@@ -284,6 +284,10 @@ jobs:
     steps:
       - custom_attach_workspace
       - run: yarn build
+      - persist_to_workspace:
+          root: *workspace_location
+          paths:
+            - dist/_*.tgz
 
   build-bazel-e2e:
     executor: action-executor
@@ -350,12 +354,14 @@ jobs:
             # Path where Arsenal Image Mounter files are downloaded.
             # Must match path in .circleci/win-ram-disk.ps1
             - ./aim
+      # Build the npm packages for the e2e tests
+      - run: yarn build
       # Run partial e2e suite on PRs only. Release branches will run the full e2e suite.
       - run:
           name: Execute E2E Tests
           command: |
             mkdir X:/ramdisk/e2e-main
-            node tests\legacy-cli\run_e2e.js --nb-shards=$env:CIRCLE_NODE_TOTAL --shard=$env:CIRCLE_NODE_INDEX --tmpdir=X:/ramdisk/e2e-main
+            node tests\legacy-cli\run_e2e.js --nb-shards=$env:CIRCLE_NODE_TOTAL --shard=$env:CIRCLE_NODE_INDEX --tmpdir=X:/ramdisk/e2e-main --package ./dist/_*.tgz
       - fail_fast
 
 workflows:

tests/legacy-cli/e2e/setup/010-local-publish.ts

@@ -1,26 +1,32 @@
+import { join } from 'path';
+import glob from 'glob';
 import { getGlobalVariable } from '../utils/env';
 import { globalNpm, extractNpmEnv } from '../utils/process';
 import { isPrereleaseCli } from '../utils/project';
 
 export default async function () {
   const testRegistry: string = getGlobalVariable('package-registry');
-  await globalNpm(
-    [
-      'run',
-      'admin',
-      '--',
-      'publish',
-      '--no-versionCheck',
-      '--no-branchCheck',
-      `--registry=${testRegistry}`,
-      '--tag',
-      isPrereleaseCli() ? 'next' : 'latest',
-    ],
-    {
-      ...extractNpmEnv(),
-      // Also set an auth token value for the local test registry which is required by npm 7+
-      // even though it is never actually used.
-      'NPM_CONFIG__AUTH': 'e2e-testing',
-    },
+
+  // Publish packages specified with --package
+  await Promise.all(
+    (getGlobalVariable('argv').package || [])
+      .flatMap((p: string) => glob.sync(p))
+      .map((p: string) =>
+        globalNpm(
+          [
+            'publish',
+            `--registry=${testRegistry}`,
+            '--tag',
+            isPrereleaseCli() ? 'next' : 'latest',
+            join(process.cwd(), p),
+          ],
+          {
+            ...extractNpmEnv(),
+            // Also set an auth token value for the local test registry which is required by npm 7+
+            // even though it is never actually used.
+            'NPM_CONFIG__AUTH': 'e2e-testing',
+          },
+        ),
+      ),
   );
 }

tests/legacy-cli/e2e/setup/BUILD.bazel

@@ -7,5 +7,6 @@ ts_library(
     visibility = ["//visibility:public"],
     deps = [
         "//tests/legacy-cli/e2e/utils",
+        "@npm//glob",
     ],
 )

tests/legacy-cli/e2e_runner.ts

@@ -34,6 +34,8 @@ Error.stackTraceLimit = Infinity;
  *   --shard          Index of this processes' shard.
  *   --tmpdir=path    Override temporary directory to use for new projects.
  *   --yarn           Use yarn as package manager.
+ *   --package=path   An npm package to be published before running tests
+ *
  * If unnamed flags are passed in, the list of tests will be filtered to include only those passed.
  */
 const argv = yargsParser(process.argv.slice(2), {
@@ -49,6 +51,7 @@ const argv = yargsParser(process.argv.slice(2), {
   ],
   string: ['devkit', 'glob', 'ignore', 'reuse', 'ng-tag', 'tmpdir', 'ng-version'],
   number: ['nb-shards', 'shard'],
+  array: ['package'],
   configuration: {
     'dot-notation': false,
     'camel-case-expansion': false,