test: unit test multiple node versions in separate jobs

Author: jbedard

.circleci/dynamic_config.yml

@@ -25,6 +25,10 @@ parameters:
 var_1: &cache_key v1-angular_devkit-14.19-{{ checksum "yarn.lock" }}
 var_1_win: &cache_key_win v1-angular_devkit-win-16.10-{{ checksum "yarn.lock" }}
 var_3: &default_nodeversion '14.19'
+var_3_major: &default_nodeversion_major '14'
+# The major version of node toolchains. See tools/toolchain_info.bzl
+# NOTE: entries in this array may be repeated elsewhere in the file, find them before adding more
+var_3_all_major: &all_nodeversion_major ['14', '16']
 # Workspace initially persisted by the `setup` job, and then enhanced by `setup-and-build-win`.
 # https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
 # https://circleci.com/blog/deep-diving-into-circleci-workspaces/
@@ -291,19 +295,39 @@ jobs:
       - custom_attach_workspace
       - run: yarn bazel build //tests/legacy-cli/...
 
-  test:
+  unit-test:
     executor: test-executor
     resource_class: xlarge
+    parameters:
+      nodeversion:
+        type: string
+        default: *default_nodeversion_major
     steps:
       - custom_attach_workspace
       - browser-tools/install-chrome
       - setup_bazel_rbe
       - run: sudo cp .circleci/bazel.rc /etc/bazel.bazelrc
-      - run:
-          command: yarn bazel:test
-          # This timeout provides time for the actual tests to timeout and report status
-          # instead of CircleCI stopping the job without test failure information.
-          no_output_timeout: 40m
+      - when:
+          # The default nodeversion runs all *excluding* other versions
+          condition:
+            equal: [*default_nodeversion_major, << parameters.nodeversion >>]
+          steps:
+            - run:
+                command: yarn bazel test --test_tag_filters=-node16,-node<< parameters.nodeversion >>-broken //packages/...
+                # This timeout provides time for the actual tests to timeout and report status
+                # instead of CircleCI stopping the job without test failure information.
+                no_output_timeout: 40m
+      - when:
+          # Non-default nodeversion runs only that specific nodeversion
+          condition:
+            not:
+              equal: [*default_nodeversion_major, << parameters.nodeversion >>]
+          steps:
+            - run:
+                command: yarn bazel test --test_tag_filters=node<< parameters.nodeversion >>,-node<< parameters.nodeversion >>-broken //packages/...
+                # This timeout provides time for the actual tests to timeout and report status
+                # instead of CircleCI stopping the job without test failure information.
+                no_output_timeout: 40m
       - fail_fast
 
   snapshot_publish:
@@ -433,7 +457,11 @@ workflows:
       # These jobs only really depend on Setup, but the build job is very quick to run (~35s) and
       # will catch any build errors before proceeding to the more lengthy and resource intensive
       # Bazel jobs.
-      - test:
+      - unit-test:
+          name: test-node<< matrix.nodeversion >>
+          matrix:
+            parameters:
+              nodeversion: *all_nodeversion_major
           requires:
             - build
 

packages/angular/cli/BUILD.bazel

@@ -156,6 +156,7 @@ ts_library(
     jasmine_node_test(
         name = "angular-cli_test_" + toolchain_name,
         srcs = [":angular-cli_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
     )
     for toolchain_name, toolchain in zip(

packages/angular/pwa/BUILD.bazel

@@ -67,6 +67,7 @@ ts_library(
     jasmine_node_test(
         name = "pwa_test_" + toolchain_name,
         srcs = [":pwa_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
     )
     for toolchain_name, toolchain in zip(

packages/angular_devkit/architect/BUILD.bazel

@@ -95,6 +95,7 @@ ts_library(
     jasmine_node_test(
         name = "architect_test_" + toolchain_name,
         srcs = [":architect_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
     )
     for toolchain_name, toolchain in zip(

packages/angular_devkit/benchmark/BUILD.bazel

@@ -60,6 +60,7 @@ ts_library(
     jasmine_node_test(
         name = "benchmark_test_" + toolchain_name,
         srcs = [":benchmark_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
         deps = [
             "@npm//jasmine",

packages/angular_devkit/build_angular/BUILD.bazel

@@ -214,6 +214,7 @@ ts_library(
     jasmine_node_test(
         name = "build_angular_test_" + toolchain_name,
         srcs = [":build_angular_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
     )
     for toolchain_name, toolchain in zip(
@@ -295,8 +296,12 @@ LARGE_SPECS = {
             "@npm//puppeteer",
             "@npm//ts-node",
         ],
-        # NB: does not run on rbe because webdriver manager uses an absolute path to chromedriver
-        "tags": ["no-remote-exec"],
+        "tags": [
+            # NB: does not run on rbe because webdriver manager uses an absolute path to chromedriver
+            "no-remote-exec",
+            # TODO: node crashes with an internal error on node16
+            "node16-broken",
+        ],
     },
     "dev-server": {
         "shards": 10,
@@ -400,7 +405,10 @@ LARGE_SPECS = {
             # These tests are resource intensive and should not be over-parallized as they will
             # compete for the resources of other parallel tests slowing everything down.
             # Ask Bazel to allocate multiple CPUs for these tests with "cpu:n" tag.
-            tags = ["cpu:2"] + LARGE_SPECS[spec].get("tags", []),
+            tags = [
+                "cpu:2",
+                toolchain_name,
+            ] + LARGE_SPECS[spec].get("tags", []),
             toolchain = toolchain,
             deps = [":build_angular_" + spec + "_test_lib"],
         )

packages/angular_devkit/build_webpack/BUILD.bazel

@@ -92,6 +92,7 @@ ts_library(
     jasmine_node_test(
         name = "build_webpack_test_" + toolchain_name,
         srcs = [":build_webpack_test_lib"],
+        tags = [toolchain_name],
         # Turns off nodejs require patches and turns on the linker, which sets up up node_modules
         # so that standard node module resolution work.
         templated_args = ["--nobazel_patch_module_resolver"],

packages/angular_devkit/core/BUILD.bazel

@@ -72,6 +72,7 @@ ts_library(
         srcs = [":core_test_lib"],
         # TODO: Audit tests to determine if tests can be run in RBE environments
         local = True,
+        tags = [toolchain_name],
         toolchain = toolchain,
         deps = [
             # @node_module: ajv

packages/angular_devkit/core/node/BUILD.bazel

@@ -57,6 +57,7 @@ ts_library(
     jasmine_node_test(
         name = "node_test_" + toolchain_name,
         srcs = [":node_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
         deps = [
             "@npm//chokidar",

packages/angular_devkit/schematics/BUILD.bazel

@@ -64,6 +64,7 @@ ts_library(
     jasmine_node_test(
         name = "schematics_test_" + toolchain_name,
         srcs = [":schematics_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
         deps = [
             "@npm//jasmine",

packages/angular_devkit/schematics/tools/BUILD.bazel

@@ -62,6 +62,7 @@ ts_library(
     jasmine_node_test(
         name = "tools_test_" + toolchain_name,
         srcs = [":tools_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
         deps = [
             "@npm//jasmine",

packages/angular_devkit/schematics_cli/BUILD.bazel

@@ -79,6 +79,7 @@ ts_library(
     jasmine_node_test(
         name = "schematics_cli_test_" + toolchain_name,
         srcs = [":schematics_cli_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
     )
     for toolchain_name, toolchain in zip(

packages/ngtools/webpack/BUILD.bazel

@@ -62,6 +62,7 @@ ts_library(
     jasmine_node_test(
         name = "webpack_test_" + toolchain_name,
         srcs = [":webpack_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
         deps = [
             "@npm//jasmine",

packages/schematics/angular/BUILD.bazel

@@ -87,6 +87,7 @@ ts_library(
     jasmine_node_test(
         name = "no_typescript_runtime_dep_test_" + toolchain_name,
         srcs = ["no_typescript_runtime_dep_spec.js"],
+        tags = [toolchain_name],
         toolchain = toolchain,
         deps = [
             ":angular",
@@ -130,6 +131,7 @@ ts_library(
     jasmine_node_test(
         name = "angular_test_" + toolchain_name,
         srcs = [":angular_test_lib"],
+        tags = [toolchain_name],
         toolchain = toolchain,
         deps = [
             "//packages/schematics/angular/third_party/github.com/Microsoft/TypeScript",

tools/toolchain_info.bzl

@@ -4,8 +4,7 @@
 # the order will match against the order in the TOOLCHAIN_VERSION list.
 TOOLCHAINS_NAMES = [
     "node14",
-    # TODO enable one we know more why there is a memory usage increase and app-shell tests work with Node.js 16.
-    # "node16",
+    "node16",
 ]
 
 # this is the list of toolchains that should be used and are registered with nodejs_register_toolchains in the WORKSPACE file
@@ -15,12 +14,11 @@ TOOLCHAINS_VERSIONS = [
         "@bazel_tools//src/conditions:darwin": "@node14_darwin_amd64//:node_toolchain",
         "@bazel_tools//src/conditions:windows": "@node14_windows_amd64//:node_toolchain",
     }),
-    # TODO enable one we know more why there is a memory usage increase and app-shell tests work with Node.js 16.
-    # select({
-    #     "@bazel_tools//src/conditions:linux_x86_64": "@node16_linux_amd64//:node_toolchain",
-    #     "@bazel_tools//src/conditions:darwin": "@node16_darwin_amd64//:node_toolchain",
-    #     "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_toolchain",
-    # }),
+    select({
+        "@bazel_tools//src/conditions:linux_x86_64": "@node16_linux_amd64//:node_toolchain",
+        "@bazel_tools//src/conditions:darwin": "@node16_darwin_amd64//:node_toolchain",
+        "@bazel_tools//src/conditions:windows": "@node16_windows_amd64//:node_toolchain",
+    }),
 ]
 
 # A default toolchain for use when only one is necessary