feat(integ-runner): support --language presets for JavaScript, TypeScript, Python and Go (#22058)

It was already possible to run tests in any language by providing `--app` and `--test-regex` directly.
This change introduces the concept of language presets that can be selected.
By default all supported languages will be detected.

Users can run integration tests for multiple languages at the same time, using the default preset configuration. 
To further customize anything, only a single language can be selected. However it's always possible to call the `integ-runner` multiple times:

```console
integ-runner --language typescript
integ-runner --language python --app="python3.2"
integ-runner --language go --test-regex=".*\.integ\.go"
```

Resolves part of #21169

----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [x] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: mrgrain

packages/@aws-cdk/integ-runner/.eslintrc.js

@@ -1,3 +1,4 @@
 const baseConfig = require('@aws-cdk/cdk-build-tools/config/eslintrc');
 baseConfig.parserOptions.project = __dirname + '/tsconfig.json';
+baseConfig.ignorePatterns = [...baseConfig.ignorePatterns, "test/language-tests/**/integ.*.ts"];
 module.exports = baseConfig;

packages/@aws-cdk/integ-runner/README.md

@@ -70,16 +70,35 @@ to be a self contained CDK app. The runner will execute the following for each f
   If this is set to `true` then the [update workflow](#update-workflow) will be disabled
 - `--app`
   The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".
+
+  Use together with `--test-regex` to fully customize how tests are run, or use with a single `--language` preset to change the command used for this language.
 - `--test-regex`
   Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.
-  
+
+  Use together with `--app` to fully customize how tests are run, or use with a single `--language` preset to change which files are detected for this language.
+- `--language`
+  The language presets to use. You can discover and run tests written in multiple languages by passing this flag multiple times (`--language typescript --language python`). Defaults to all supported languages. Currently supported language presets are:
+  - `javascript`:
+    - File RegExp: `^integ\..*\.js$`
+    - App run command: `node {filePath}`
+  - `typescript`:\
+    Note that for TypeScript files compiled to JavaScript, the JS tests will take precedence and the TS ones won't be evaluated.
+    - File RegExp: `^integ\..*(?<!\.d)\.ts$`
+    - App run command: `node -r ts-node/register {filePath}`
+  - `python`:
+    - File RegExp: `^integ_.*\.py$`
+    - App run command: `python {filePath}`
+  - `go`:
+    - File RegExp: `^integ_.*\.go$`
+    - App run command: `go run {filePath}`
+
 Example:
 
 ```bash
-integ-runner --update-on-failed --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./
+integ-runner --update-on-failed --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./ --language python
 ```
 
-This will search for integration tests recursively from the current directory and then execute them in parallel across `us-east-1`, `us-east-2`, & `us-west-2`.
+This will search for python integration tests recursively from the current directory and then execute them in parallel across `us-east-1`, `us-east-2`, & `us-west-2`.
 
 If you are providing a list of tests to execute, either as CLI arguments or from a file, the name of the test needs to be relative to the `directory`.
 For example, if there is a test `aws-iam/test/integ.policy.js` and the current working directory is `aws-iam` you would provide `integ.policy.js`

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -17,7 +17,7 @@ export function parseCliArgs(args: string[] = []) {
     .usage('Usage: integ-runner [TEST...]')
     .option('config', {
       config: true,
-      configParser: IntegrationTests.configFromFile,
+      configParser: configFromFile,
       default: 'integ.config.json',
       desc: 'Load options from a JSON config file. Options provided as CLI arguments take precedent.',
     })
@@ -35,6 +35,13 @@ export function parseCliArgs(args: string[] = []) {
     .options('from-file', { type: 'string', desc: 'Read TEST names from a file (one TEST per line)' })
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
+    .option('language', {
+      alias: 'l',
+      default: ['javascript', 'typescript', 'python', 'go'],
+      choices: ['javascript', 'typescript', 'python', 'go'],
+      type: 'array',
+      desc: 'Use these presets to run integration tests for the selected languages',
+    })
     .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
     .option('test-regex', { type: 'array', desc: 'Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.', default: [] })
     .strict()
@@ -80,19 +87,15 @@ export function parseCliArgs(args: string[] = []) {
     force: argv.force as boolean,
     dryRun: argv['dry-run'] as boolean,
     disableUpdateWorkflow: argv['disable-update-workflow'] as boolean,
+    language: arrayFromYargs(argv.language),
   };
 }
 
 
 export async function main(args: string[]) {
   const options = parseCliArgs(args);
 
-  const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliArgs({
-    app: options.app,
-    testRegex: options.testRegex,
-    tests: options.tests,
-    exclude: options.exclude,
-  });
+  const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliOptions(options);
 
   // List only prints the discoverd tests
   if (options.list) {
@@ -227,3 +230,21 @@ export function cli(args: string[] = process.argv.slice(2)) {
     process.exitCode = 1;
   });
 }
+
+/**
+ * Read CLI options from a config file if provided.
+ *
+ * @param fileName
+ * @returns parsed CLI config options
+ */
+function configFromFile(fileName?: string): Record<string, any> {
+  if (!fileName) {
+    return {};
+  }
+
+  try {
+    return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
+  } catch {
+    return {};
+  }
+}

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -159,42 +159,112 @@ export interface IntegrationTestsDiscoveryOptions {
   readonly tests?: string[];
 
   /**
-    * Detect integration test files matching any of these JavaScript regex patterns.
-    *
-    * @default
-    */
-  readonly testRegex?: string[];
-
-  /**
-   * The CLI command used to run this test.
-   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
+   * A map of of the app commands to run integration tests with,
+   * and the regex patterns matching the integration test files each app command.
    *
-   * @default - test run command will be `node {filePath}`
+   * If the app command contains {filePath}, the test file names will be substituted at that place in the command for each run.
    */
-  readonly app?: string;
+  readonly testCases: {
+    [app: string]: string[]
+  }
 }
 
+/**
+ * Returns the name of the Python executable for the current OS
+ */
+function pythonExecutable() {
+  let python = 'python3';
+  if (process.platform === 'win32') {
+    python = 'python';
+  }
+  return python;
+}
 
 /**
  * Discover integration tests
  */
 export class IntegrationTests {
+  constructor(private readonly directory: string) {}
+
   /**
-   * Return configuration options from a file
+   * Get integration tests discovery options from CLI options
    */
-  public static configFromFile(fileName?: string): Record<string, any> {
-    if (!fileName) {
-      return {};
+  public async fromCliOptions(options: {
+    app?: string;
+    exclude?: boolean,
+    language?: string[],
+    testRegex?: string[],
+    tests?: string[],
+  }): Promise<IntegTest[]> {
+    const baseOptions = {
+      tests: options.tests,
+      exclude: options.exclude,
+    };
+
+    // Explicitly set both, app and test-regex
+    if (options.app && options.testRegex) {
+      return this.discover({
+        testCases: {
+          [options.app]: options.testRegex,
+        },
+        ...baseOptions,
+      });
+    }
+
+    // Use the selected presets
+    if (!options.app && !options.testRegex) {
+      // Only case with multiple languages, i.e. the only time we need to check the special case
+      const ignoreUncompiledTypeScript = options.language?.includes('javascript') && options.language?.includes('typescript');
+
+      return this.discover({
+        testCases: this.getLanguagePresets(options.language),
+        ...baseOptions,
+      }, ignoreUncompiledTypeScript);
     }
 
-    try {
-      return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
-    } catch {
-      return {};
+    // Only one of app or test-regex is set, with a single preset selected
+    // => override either app or test-regex
+    if (options.language?.length === 1) {
+      const [presetApp, presetTestRegex] = this.getLanguagePreset(options.language[0]);
+      return this.discover({
+        testCases: {
+          [options.app ?? presetApp]: options.testRegex ?? presetTestRegex,
+        },
+        ...baseOptions,
+      });
     }
+
+    // Only one of app or test-regex is set, with multiple presets
+    // => impossible to resolve
+    const option = options.app ? '--app' : '--test-regex';
+    throw new Error(`Only a single "--language" can be used with "${option}". Alternatively provide both "--app" and "--test-regex" to fully customize the configuration.`);
+  }
+
+  /**
+   * Get the default configuration for a language
+   */
+  private getLanguagePreset(language: string) {
+    const languagePresets: {
+      [language: string]: [string, string[]]
+    } = {
+      javascript: ['node {filePath}', ['^integ\\..*\\.js$']],
+      typescript: ['node -r ts-node/register {filePath}', ['^integ\\.(?!.*\\.d\\.ts$).*\\.ts$']],
+      python: [`${pythonExecutable()} {filePath}`, ['^integ_.*\\.py$']],
+      go: ['go run {filePath}', ['^integ_.*\\.go$']],
+    };
+
+    return languagePresets[language];
   }
 
-  constructor(private readonly directory: string) {
+  /**
+   * Get the config for all selected languages
+   */
+  private getLanguagePresets(languages: string[] = []) {
+    return Object.fromEntries(
+      languages
+        .map(language => this.getLanguagePreset(language))
+        .filter(Boolean),
+    );
   }
 
   /**
@@ -209,7 +279,6 @@ export class IntegrationTests {
       return discoveredTests;
     }
 
-
     const allTests = discoveredTests.filter(t => {
       const matches = requestedTests.some(pattern => t.matches(pattern));
       return matches !== !!exclude; // Looks weird but is equal to (matches && !exclude) || (!matches && exclude)
@@ -237,33 +306,41 @@ export class IntegrationTests {
    * @param tests Tests to include or exclude, undefined means include all tests.
    * @param exclude Whether the 'tests' list is inclusive or exclusive (inclusive by default).
    */
-  public async fromCliArgs(options: IntegrationTestsDiscoveryOptions = {}): Promise<IntegTest[]> {
-    return this.discover(options);
-  }
-
-  private async discover(options: IntegrationTestsDiscoveryOptions): Promise<IntegTest[]> {
-    const patterns = options.testRegex ?? ['^integ\\..*\\.js$'];
-
+  private async discover(options: IntegrationTestsDiscoveryOptions, ignoreUncompiledTypeScript: boolean = false): Promise<IntegTest[]> {
     const files = await this.readTree();
-    const integs = files.filter(fileName => patterns.some((p) => {
-      const regex = new RegExp(p);
-      return regex.test(fileName) || regex.test(path.basename(fileName));
-    }));
-
-    return this.request(integs, options);
-  }
-
-  private request(files: string[], options: IntegrationTestsDiscoveryOptions): IntegTest[] {
-    const discoveredTests = files.map(fileName => new IntegTest({
-      discoveryRoot: this.directory,
-      fileName,
-      appCommand: options.app,
-    }));
 
+    const testCases = Object.entries(options.testCases)
+      .flatMap(([appCommand, patterns]) => files
+        .filter(fileName => patterns.some((pattern) => {
+          const regex = new RegExp(pattern);
+          return regex.test(fileName) || regex.test(path.basename(fileName));
+        }))
+        .map(fileName => new IntegTest({
+          discoveryRoot: this.directory,
+          fileName,
+          appCommand,
+        })),
+      );
+
+    const discoveredTests = ignoreUncompiledTypeScript ? this.filterUncompiledTypeScript(testCases) : testCases;
 
     return this.filterTests(discoveredTests, options.tests, options.exclude);
   }
 
+  private filterUncompiledTypeScript(testCases: IntegTest[]): IntegTest[] {
+    const jsTestCases = testCases.filter(t => t.fileName.endsWith('.js'));
+
+    return testCases
+      // Remove all TypeScript test cases (ending in .ts)
+      // for which a compiled version is present (same name, ending in .js)
+      .filter((tsCandidate) => {
+        if (!tsCandidate.fileName.endsWith('.ts')) {
+          return true;
+        }
+        return jsTestCases.findIndex(jsTest => jsTest.testName === tsCandidate.testName) === -1;
+      });
+  }
+
   private async readTree(): Promise<string[]> {
     const ret = new Array<string>();
 

packages/@aws-cdk/integ-runner/package.json

@@ -14,6 +14,7 @@
     "awslint": "cdk-awslint",
     "pkglint": "pkglint -f",
     "test": "cdk-test",
+    "integ": "integ-runner",
     "watch": "cdk-watch",
     "build+test": "yarn build && yarn test",
     "build+test+package": "yarn build+test && yarn package",
@@ -52,15 +53,19 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@aws-cdk/cdk-build-tools": "0.0.0",
-    "@types/mock-fs": "^4.13.1",
-    "mock-fs": "^4.14.0",
+    "@aws-cdk/core": "0.0.0",
+    "@aws-cdk/integ-tests": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",
     "@types/fs-extra": "^8.1.2",
     "@types/jest": "^27.5.2",
+    "@types/mock-fs": "^4.13.1",
     "@types/node": "^14.18.34",
     "@types/workerpool": "^6.1.0",
     "@types/yargs": "^15.0.14",
-    "jest": "^27.5.1"
+    "constructs": "^10.0.0",
+    "mock-fs": "^4.14.0",
+    "jest": "^27.5.1",
+    "ts-node": "^10.9.1"
   },
   "dependencies": {
     "@aws-cdk/cloud-assembly-schema": "0.0.0",