ci: add flake jail

Author: filipesilva

.circleci/config.yml

@@ -241,6 +241,16 @@ jobs:
       - attach_workspace: *attach_options
       - run: yarn build
 
+  # This is where we put all the misbehaving and flaky tests so we can fine-tune their conditions 
+  # and rerun them faster.
+  flake-jail:
+    executor: action-executor
+    steps:
+      - attach_workspace: *attach_options
+      - run: yarn webdriver-update
+      - run: yarn test-large --full --flakey
+      - run: yarn test-large --full --flakey --ivy=true
+
   build-bazel:
     executor: action-executor
     resource_class: xlarge
@@ -379,6 +389,9 @@ workflows:
       - test-browsers:
           requires:
             - build
+      - flake-jail:
+          requires:
+            - build
       - snapshot_publish:
           <<: *ignore_pull_requests
           requires:

scripts/test.ts

@@ -22,6 +22,12 @@ import { packages } from '../lib/packages';
 const codeMap = require('../lib/istanbul-local').codeMap;
 const Jasmine = require('jasmine');
 
+const knownFlakes = [
+  // Rebuild tests in test-large are flakey if not run as the first suite.
+  // https://github.com/angular/angular-cli/pull/15204
+  'packages/angular_devkit/build_angular/test/browser/rebuild_spec_large.ts',
+];
+
 const projectBaseDir = join(__dirname, '..');
 require('source-map-support').install({
   hookRequire: true,
@@ -265,6 +271,9 @@ export default function(args: ParsedArgs, logger: logging.Logger) {
     }
   }
 
+  // Filter in/out flakes according to the --flakey flag.
+  tests = tests.filter(test => !!args.flakey == knownFlakes.includes(test.replace(/[\/\\]/g, '/')));
+
   if (args.shard !== undefined) {
     // Remove tests that are not part of this shard.
     const shardId = args['shard'];