feat(integ-runner): support custom --test-regex to match integ test files (#22786)

Follow-up to #22761. To support other languages than JavaScript (see #22521) we need to be able to detect test files with any patterns. With this PR, users can specify a number of custom `--test-regex` patterns that will bed used to discover integration test files. Together with `--app` this can already be used to run integ tests in arbitrary languages.

Example usage: `integ-runner --app="python3 {filePath}" --test-regex="^integ_.*\.py$"`

Also contains a minor refactor to make `--app` available via `IntegrationTests.fromFile()`. This is in preparation of an upcoming change to reestablish support for an integration test config file.

----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: mrgrain

packages/@aws-cdk/integ-runner/README.md

@@ -70,6 +70,9 @@ to be a self contained CDK app. The runner will execute the following for each f
   If this is set to `true` then the [update workflow](#update-workflow) will be disabled
 - `--app`
   The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".
+- `--test-regex`
+  Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.
+  
 Example:
 
 ```bash

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -13,7 +13,7 @@ import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWor
 const yargs = require('yargs');
 
 
-async function main() {
+export async function main(args: string[]) {
   const argv = yargs
     .usage('Usage: integ-runner [TEST...]')
     .option('list', { type: 'boolean', default: false, desc: 'List tests instead of running them' })
@@ -31,14 +31,16 @@ async function main() {
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
     .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
+    .option('test-regex', { type: 'array', desc: 'Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.', default: [] })
     .strict()
-    .argv;
+    .parse(args);
 
   const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), {
     maxWorkers: argv['max-workers'],
   });
 
   // list of integration tests that will be executed
+  const testRegex = arrayFromYargs(argv['test-regex']);
   const testsToRun: IntegTestWorkerConfig[] = [];
   const destructiveChanges: DestructiveChange[] = [];
   const testsFromArgs: IntegTest[] = [];
@@ -48,6 +50,7 @@ async function main() {
   const runUpdateOnFailed = argv['update-on-failed'] ?? false;
   const fromFile: string | undefined = argv['from-file'];
   const exclude: boolean = argv.exclude;
+  const app: string | undefined = argv.app;
 
   let failedSnapshots: IntegTestWorkerConfig[] = [];
   if (argv['max-workers'] < testRegions.length * (profiles ?? [1]).length) {
@@ -57,7 +60,7 @@ async function main() {
   let testsSucceeded = false;
   try {
     if (argv.list) {
-      const tests = await new IntegrationTests(argv.directory).fromCliArgs();
+      const tests = await new IntegrationTests(argv.directory).fromCliArgs({ testRegex, app });
       process.stdout.write(tests.map(t => t.discoveryRelativeFileName).join('\n') + '\n');
       return;
     }
@@ -69,15 +72,19 @@ async function main() {
       ? (await fs.readFile(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
       : (argv._.length > 0 ? argv._ : undefined); // 'undefined' means no request
 
-    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs(requestedTests, exclude)));
+    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs({
+      app,
+      testRegex,
+      tests: requestedTests,
+      exclude,
+    })));
 
     // always run snapshot tests, but if '--force' is passed then
     // run integration tests on all failed tests, not just those that
     // failed snapshot tests
     failedSnapshots = await runSnapshotTests(pool, testsFromArgs, {
       retain: argv['inspect-failures'],
       verbose: Boolean(argv.verbose),
-      appCommand: argv.app,
     });
     for (const failure of failedSnapshots) {
       destructiveChanges.push(...failure.destructiveChanges ?? []);
@@ -101,7 +108,6 @@ async function main() {
         dryRun: argv['dry-run'],
         verbosity: argv.verbose,
         updateWorkflow: !argv['disable-update-workflow'],
-        appCommand: argv.app,
       });
       testsSucceeded = success;
 
@@ -184,8 +190,8 @@ function mergeTests(testFromArgs: IntegTestInfo[], failedSnapshotTests: IntegTes
   return final;
 }
 
-export function cli() {
-  main().then().catch(err => {
+export function cli(args: string[] = process.argv.slice(2)) {
+  main(args).then().catch(err => {
     logger.error(err);
     process.exitCode = 1;
   });

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -23,6 +23,14 @@ export interface IntegTestInfo {
    * Path is relative to the current working directory.
    */
   readonly discoveryRoot: string;
+
+  /**
+   * The CLI command used to run this test.
+   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
+   *
+   * @default - test run command will be `node {filePath}`
+   */
+  readonly appCommand?: string;
 }
 
 /**
@@ -79,7 +87,16 @@ export class IntegTest {
    */
   public readonly temporaryOutputDir: string;
 
+  /**
+   * The CLI command used to run this test.
+   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
+   *
+   * @default - test run command will be `node {filePath}`
+   */
+  readonly appCommand: string;
+
   constructor(public readonly info: IntegTestInfo) {
+    this.appCommand = info.appCommand ?? 'node {filePath}';
     this.absoluteFileName = path.resolve(info.fileName);
     this.fileName = path.relative(process.cwd(), info.fileName);
 
@@ -123,10 +140,9 @@ export class IntegTest {
 }
 
 /**
- * The list of tests to run can be provided in a file
- * instead of as command line arguments.
+ * Configuration options how integration test files are discovered
  */
-export interface IntegrationTestFileConfig {
+export interface IntegrationTestsDiscoveryOptions {
   /**
    * If this is set to true then the list of tests
    * provided will be excluded
@@ -135,6 +151,35 @@ export interface IntegrationTestFileConfig {
    */
   readonly exclude?: boolean;
 
+  /**
+    * List of tests to include (or exclude if `exclude=true`)
+    *
+    * @default - all matched files
+    */
+  readonly tests?: string[];
+
+  /**
+    * Detect integration test files matching any of these JavaScript regex patterns.
+    *
+    * @default
+    */
+  readonly testRegex?: string[];
+
+  /**
+   * The CLI command used to run this test.
+   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
+   *
+   * @default - test run command will be `node {filePath}`
+   */
+  readonly app?: string;
+}
+
+
+/**
+ * The list of tests to run can be provided in a file
+ * instead of as command line arguments.
+ */
+export interface IntegrationTestFileConfig extends IntegrationTestsDiscoveryOptions {
   /**
    * List of tests to include (or exclude if `exclude=true`)
    */
@@ -154,11 +199,8 @@ export class IntegrationTests {
    */
   public async fromFile(fileName: string): Promise<IntegTest[]> {
     const file: IntegrationTestFileConfig = JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
-    const foundTests = await this.discover();
-
-    const allTests = this.filterTests(foundTests, file.tests, file.exclude);
 
-    return allTests;
+    return this.discover(file);
   }
 
   /**
@@ -201,22 +243,31 @@ export class IntegrationTests {
    * @param tests Tests to include or exclude, undefined means include all tests.
    * @param exclude Whether the 'tests' list is inclusive or exclusive (inclusive by default).
    */
-  public async fromCliArgs(tests?: string[], exclude?: boolean): Promise<IntegTest[]> {
-    const discoveredTests = await this.discover();
-
-    const allTests = this.filterTests(discoveredTests, tests, exclude);
-
-    return allTests;
+  public async fromCliArgs(options: IntegrationTestsDiscoveryOptions = {}): Promise<IntegTest[]> {
+    return this.discover(options);
   }
 
-  private async discover(): Promise<IntegTest[]> {
+  private async discover(options: IntegrationTestsDiscoveryOptions): Promise<IntegTest[]> {
+    const patterns = options.testRegex ?? ['^integ\\..*\\.js$'];
+
     const files = await this.readTree();
-    const integs = files.filter(fileName => path.basename(fileName).startsWith('integ.') && path.basename(fileName).endsWith('.js'));
-    return this.request(integs);
+    const integs = files.filter(fileName => patterns.some((p) => {
+      const regex = new RegExp(p);
+      return regex.test(fileName) || regex.test(path.basename(fileName));
+    }));
+
+    return this.request(integs, options);
   }
 
-  private request(files: string[]): IntegTest[] {
-    return files.map(fileName => new IntegTest({ discoveryRoot: this.directory, fileName }));
+  private request(files: string[], options: IntegrationTestsDiscoveryOptions): IntegTest[] {
+    const discoveredTests = files.map(fileName => new IntegTest({
+      discoveryRoot: this.directory,
+      fileName,
+      appCommand: options.app,
+    }));
+
+
+    return this.filterTests(discoveredTests, options.tests, options.exclude);
   }
 
   private async readTree(): Promise<string[]> {
@@ -228,7 +279,7 @@ export class IntegrationTests {
         const fullPath = path.join(dir, file);
         const statf = await fs.stat(fullPath);
         if (statf.isFile()) { ret.push(fullPath); }
-        if (statf.isDirectory()) { await recurse(path.join(fullPath)); }
+        if (statf.isDirectory()) { await recurse(fullPath); }
       }
     }
 

packages/@aws-cdk/integ-runner/lib/runner/runner-base.ts

@@ -49,14 +49,6 @@ export interface IntegRunnerOptions {
    */
   readonly cdk?: ICdk;
 
-  /**
-   * You can specify a custom run command, and it will be applied to all test files.
-   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
-   *
-   * @default - test run command will be `node {filePath}`
-   */
-  readonly appCommand?: string;
-
   /**
    * Show output from running integration tests
    *
@@ -159,7 +151,7 @@ export abstract class IntegRunner {
     });
     this.cdkOutDir = options.integOutDir ?? this.test.temporaryOutputDir;
 
-    const testRunCommand = options.appCommand ?? 'node {filePath}';
+    const testRunCommand = this.test.appCommand;
     this.cdkApp = testRunCommand.replace('{filePath}', path.relative(this.directory, this.test.fileName));
 
     this.profile = options.profile;

packages/@aws-cdk/integ-runner/lib/workers/common.ts

@@ -103,13 +103,6 @@ export interface SnapshotVerificationOptions {
    * @default false
    */
   readonly verbose?: boolean;
-
-  /**
-   * The CLI command used to run the test files.
-   *
-   * @default - test run command will be `node {filePath}`
-   */
-  readonly appCommand?: string;
 }
 
 /**
@@ -169,13 +162,6 @@ export interface IntegTestOptions {
    * @default true
    */
   readonly updateWorkflow?: boolean;
-
-  /**
-   * The CLI command used to run the test files.
-   *
-   * @default - test run command will be `node {filePath}`
-   */
-  readonly appCommand?: string;
 }
 
 /**

packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts

@@ -27,7 +27,6 @@ export function integTestWorker(request: IntegTestBatchRequest): IntegTestWorker
         env: {
           AWS_REGION: request.region,
         },
-        appCommand: request.appCommand,
         showOutput: verbosity >= 2,
       }, testInfo.destructiveChanges);
 
@@ -106,7 +105,7 @@ export function snapshotTestWorker(testInfo: IntegTestInfo, options: SnapshotVer
   }, 60_000);
 
   try {
-    const runner = new IntegSnapshotRunner({ test, appCommand: options.appCommand });
+    const runner = new IntegSnapshotRunner({ test });
     if (!runner.hasSnapshot()) {
       workerpool.workerEmit({
         reason: DiagnosticReason.NO_SNAPSHOT,

packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts

@@ -135,7 +135,6 @@ export async function runIntegrationTestsInParallel(
         dryRun: options.dryRun,
         verbosity: options.verbosity,
         updateWorkflow: options.updateWorkflow,
-        appCommand: options.appCommand,
       }], {
         on: printResults,
       });

packages/@aws-cdk/integ-runner/test/cli.test.ts

@@ -0,0 +1,44 @@
+import * as path from 'path';
+import { main } from '../lib/cli';
+
+describe('CLI', () => {
+  const currentCwd = process.cwd();
+  beforeAll(() => {
+    process.chdir(path.join(__dirname, '..'));
+  });
+  afterAll(() => {
+    process.chdir(currentCwd);
+  });
+
+  let stdoutMock: jest.SpyInstance;
+  beforeEach(() => {
+    stdoutMock = jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; });
+  });
+  afterEach(() => {
+    stdoutMock.mockRestore();
+  });
+
+  test('find by default pattern', async () => {
+    await main(['--list', '--directory=test/test-data']);
+
+    // Expect nothing to be found since this directory doesn't contain files with the default pattern
+    expect(stdoutMock.mock.calls).toEqual([['\n']]);
+  });
+
+  test('find by custom pattern', async () => {
+    await main(['--list', '--directory=test/test-data', '--test-regex="^xxxxx\..*\.js$"']);
+
+    expect(stdoutMock.mock.calls).toEqual([[
+      [
+        'xxxxx.integ-test1.js',
+        'xxxxx.integ-test2.js',
+        'xxxxx.test-with-new-assets-diff.js',
+        'xxxxx.test-with-new-assets.js',
+        'xxxxx.test-with-snapshot-assets-diff.js',
+        'xxxxx.test-with-snapshot-assets.js',
+        'xxxxx.test-with-snapshot.js',
+        '',
+      ].join('\n'),
+    ]]);
+  });
+});

packages/@aws-cdk/integ-runner/test/runner/integ-test-runner.test.ts

@@ -567,8 +567,8 @@ describe('IntegTest runIntegTests', () => {
       test: new IntegTest({
         fileName: 'test/test-data/xxxxx.test-with-snapshot.js',
         discoveryRoot: 'test/test-data',
+        appCommand: 'node --no-warnings {filePath}',
       }),
-      appCommand: 'node --no-warnings {filePath}',
     });
     integTest.runIntegTestCase({
       testCaseName: 'xxxxx.test-with-snapshot',