feat(integ-runner): support config file (#22937)

Enables support for yarg's config file, replacing the current unused `IntegrationTests.fromFile()` method.

This also fixes a bug where `--list` would ignore test provided as args and via `--from-file` and just list all tests.

----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: mrgrain

packages/@aws-cdk/integ-runner/README.md

@@ -85,7 +85,7 @@ If you are providing a list of tests to execute, either as CLI arguments or from
 For example, if there is a test `aws-iam/test/integ.policy.js` and the current working directory is `aws-iam` you would provide `integ.policy.js`
 
 ```bash
-yarn integ integ.policy.js
+integ-runner integ.policy.js
 ```
 
 ### Common Workflow
@@ -121,7 +121,7 @@ Snapshot Results:
 
 Tests:    1 failed, 9 total
 Error: Some snapshot tests failed!
-To re-run failed tests run: yarn integ-runner --update-on-failed
+To re-run failed tests run: integ-runner --update-on-failed
     at main (packages/@aws-cdk/integ-runner/lib/cli.js:90:15)
 error Command failed with exit code 1. 
 ```
@@ -197,7 +197,7 @@ If you are adding a new test which creates a new snapshot then you should run th
 For example, if you are working on a new test `integ.new-test.js` then you would run:
 
 ```bash
-yarn integ --update-on-failed --disable-update-workflow integ.new-test.js
+integ-runner --update-on-failed --disable-update-workflow integ.new-test.js
 ```
 
 This is because for a new test we do not need to test the update workflow (there is nothing to update).
@@ -210,3 +210,25 @@ See [@aws-cdk/cloud-assembly-schema/lib/integ-tests/schema.ts](../cloud-assembly
 
 See the `@aws-cdk/integ-tests` module for information on how to define
 integration tests for the runner to exercise.
+
+### Config file
+
+All options can be configured via the `integ.config.json` configuration file in the current working directory.
+
+```json
+{
+  "maxWorkers": 10,
+  "parallelRegions": [
+    "eu-west-1",
+    "ap-southeast-2"
+  ]
+}
+```
+
+Available options can be listed by running the following command:
+
+```sh
+integ-runner --help
+```
+
+To use a different config file, provide the `--config` command-line option.

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -1,21 +1,26 @@
 // Exercise all integ stacks and if they deploy, update the expected synth files
-import { promises as fs } from 'fs';
+import * as fs from 'fs';
 import * as path from 'path';
 import * as chalk from 'chalk';
 import * as workerpool from 'workerpool';
 import * as logger from './logger';
-import { IntegrationTests, IntegTestInfo, IntegTest } from './runner/integration-tests';
+import { IntegrationTests, IntegTestInfo } from './runner/integration-tests';
 import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers';
 
 // https://github.com/yargs/yargs/issues/1929
 // https://github.com/evanw/esbuild/issues/1492
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const yargs = require('yargs');
 
-
-export async function main(args: string[]) {
+export function parseCliArgs(args: string[] = []) {
   const argv = yargs
     .usage('Usage: integ-runner [TEST...]')
+    .option('config', {
+      config: true,
+      configParser: IntegrationTests.configFromFile,
+      default: 'integ.config.json',
+      desc: 'Load options from a JSON config file. Options provided as CLI arguments take precedent.',
+    })
     .option('list', { type: 'boolean', default: false, desc: 'List tests instead of running them' })
     .option('clean', { type: 'boolean', default: true, desc: 'Skips stack clean up after test is completed (use --no-clean to negate)' })
     .option('verbose', { type: 'boolean', default: false, alias: 'v', count: true, desc: 'Verbose logs and metrics on integration tests durations (specify multiple times to increase verbosity)' })
@@ -35,61 +40,87 @@ export async function main(args: string[]) {
     .strict()
     .parse(args);
 
-  const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), {
-    maxWorkers: argv['max-workers'],
-  });
-
-  // list of integration tests that will be executed
-  const testRegex = arrayFromYargs(argv['test-regex']);
-  const testsToRun: IntegTestWorkerConfig[] = [];
-  const destructiveChanges: DestructiveChange[] = [];
-  const testsFromArgs: IntegTest[] = [];
+  const tests: string[] = argv._;
   const parallelRegions = arrayFromYargs(argv['parallel-regions']);
   const testRegions: string[] = parallelRegions ?? ['us-east-1', 'us-east-2', 'us-west-2'];
   const profiles = arrayFromYargs(argv.profiles);
-  const runUpdateOnFailed = argv['update-on-failed'] ?? false;
   const fromFile: string | undefined = argv['from-file'];
-  const exclude: boolean = argv.exclude;
-  const app: string | undefined = argv.app;
+  const maxWorkers: number = argv['max-workers'];
+  const verbosity: number = argv.verbose;
+  const verbose: boolean = verbosity >= 1;
 
-  let failedSnapshots: IntegTestWorkerConfig[] = [];
-  if (argv['max-workers'] < testRegions.length * (profiles ?? [1]).length) {
-    logger.warning('You are attempting to run %s tests in parallel, but only have %s workers. Not all of your profiles+regions will be utilized', argv.profiles * argv['parallel-regions'], argv['max-workers']);
+  const numTests = testRegions.length * (profiles ?? [1]).length;
+  if (maxWorkers < numTests) {
+    logger.warning('You are attempting to run %s tests in parallel, but only have %s workers. Not all of your profiles+regions will be utilized', numTests, maxWorkers);
   }
 
-  let testsSucceeded = false;
-  try {
-    if (argv.list) {
-      const tests = await new IntegrationTests(argv.directory).fromCliArgs({ testRegex, app });
-      process.stdout.write(tests.map(t => t.discoveryRelativeFileName).join('\n') + '\n');
-      return;
-    }
+  if (tests.length > 0 && fromFile) {
+    throw new Error('A list of tests cannot be provided if "--from-file" is provided');
+  }
+  const requestedTests = fromFile
+    ? (fs.readFileSync(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
+    : (tests.length > 0 ? tests : undefined); // 'undefined' means no request
+
+  return {
+    tests: requestedTests,
+    app: argv.app as (string | undefined),
+    testRegex: arrayFromYargs(argv['test-regex']),
+    testRegions,
+    profiles,
+    runUpdateOnFailed: (argv['update-on-failed'] ?? false) as boolean,
+    fromFile,
+    exclude: argv.exclude as boolean,
+    maxWorkers,
+    list: argv.list as boolean,
+    directory: argv.directory as string,
+    inspectFailures: argv['inspect-failures'] as boolean,
+    verbosity,
+    verbose,
+    clean: argv.clean as boolean,
+    force: argv.force as boolean,
+    dryRun: argv['dry-run'] as boolean,
+    disableUpdateWorkflow: argv['disable-update-workflow'] as boolean,
+  };
+}
 
-    if (argv._.length > 0 && fromFile) {
-      throw new Error('A list of tests cannot be provided if "--from-file" is provided');
-    }
-    const requestedTests = fromFile
-      ? (await fs.readFile(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
-      : (argv._.length > 0 ? argv._ : undefined); // 'undefined' means no request
 
-    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs({
-      app,
-      testRegex,
-      tests: requestedTests,
-      exclude,
-    })));
+export async function main(args: string[]) {
+  const options = parseCliArgs(args);
+
+  const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliArgs({
+    app: options.app,
+    testRegex: options.testRegex,
+    tests: options.tests,
+    exclude: options.exclude,
+  });
+
+  // List only prints the discoverd tests
+  if (options.list) {
+    process.stdout.write(testsFromArgs.map(t => t.discoveryRelativeFileName).join('\n') + '\n');
+    return;
+  }
+
+  const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), {
+    maxWorkers: options.maxWorkers,
+  });
 
+  const testsToRun: IntegTestWorkerConfig[] = [];
+  const destructiveChanges: DestructiveChange[] = [];
+  let failedSnapshots: IntegTestWorkerConfig[] = [];
+  let testsSucceeded = false;
+
+  try {
     // always run snapshot tests, but if '--force' is passed then
     // run integration tests on all failed tests, not just those that
     // failed snapshot tests
     failedSnapshots = await runSnapshotTests(pool, testsFromArgs, {
-      retain: argv['inspect-failures'],
-      verbose: Boolean(argv.verbose),
+      retain: options.inspectFailures,
+      verbose: options.verbose,
     });
     for (const failure of failedSnapshots) {
       destructiveChanges.push(...failure.destructiveChanges ?? []);
     }
-    if (!argv.force) {
+    if (!options.force) {
       testsToRun.push(...failedSnapshots);
     } else {
       // if any of the test failed snapshot tests, keep those results
@@ -98,25 +129,25 @@ export async function main(args: string[]) {
     }
 
     // run integration tests if `--update-on-failed` OR `--force` is used
-    if (runUpdateOnFailed || argv.force) {
+    if (options.runUpdateOnFailed || options.force) {
       const { success, metrics } = await runIntegrationTests({
         pool,
         tests: testsToRun,
-        regions: testRegions,
-        profiles,
-        clean: argv.clean,
-        dryRun: argv['dry-run'],
-        verbosity: argv.verbose,
-        updateWorkflow: !argv['disable-update-workflow'],
+        regions: options.testRegions,
+        profiles: options.profiles,
+        clean: options.clean,
+        dryRun: options.dryRun,
+        verbosity: options.verbosity,
+        updateWorkflow: !options.disableUpdateWorkflow,
       });
       testsSucceeded = success;
 
 
-      if (argv.clean === false) {
+      if (options.clean === false) {
         logger.warning('Not cleaning up stacks since "--no-clean" was used');
       }
 
-      if (Boolean(argv.verbose)) {
+      if (Boolean(options.verbose)) {
         printMetrics(metrics);
       }
 
@@ -134,8 +165,8 @@ export async function main(args: string[]) {
   }
   if (failedSnapshots.length > 0) {
     let message = '';
-    if (!runUpdateOnFailed) {
-      message = 'To re-run failed tests run: yarn integ-runner --update-on-failed';
+    if (!options.runUpdateOnFailed) {
+      message = 'To re-run failed tests run: integ-runner --update-on-failed';
     }
     if (!testsSucceeded) {
       throw new Error(`Some tests failed!\n${message}`);

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -175,32 +175,26 @@ export interface IntegrationTestsDiscoveryOptions {
 }
 
 
-/**
- * The list of tests to run can be provided in a file
- * instead of as command line arguments.
- */
-export interface IntegrationTestFileConfig extends IntegrationTestsDiscoveryOptions {
-  /**
-   * List of tests to include (or exclude if `exclude=true`)
-   */
-  readonly tests: string[];
-}
-
 /**
  * Discover integration tests
  */
 export class IntegrationTests {
-  constructor(private readonly directory: string) {
-  }
-
   /**
-   * Takes a file name of a file that contains a list of test
-   * to either run or exclude and returns a list of Integration Tests to run
+   * Return configuration options from a file
    */
-  public async fromFile(fileName: string): Promise<IntegTest[]> {
-    const file: IntegrationTestFileConfig = JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
+  public static configFromFile(fileName?: string): Record<string, any> {
+    if (!fileName) {
+      return {};
+    }
 
-    return this.discover(file);
+    try {
+      return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
+    } catch {
+      return {};
+    }
+  }
+
+  constructor(private readonly directory: string) {
   }
 
   /**

packages/@aws-cdk/integ-runner/lib/utils.ts

@@ -75,6 +75,7 @@ export class WorkList<A> {
 
   public done() {
     this.remaining.clear();
+    this.stopTimer();
   }
 
   private stopTimer() {