chore: forward merge 'master' into 'v2-main' (#19999)

Automated action from aws/cdk-ops

Author: mergify[bot]

INTEGRATION_TESTS.md

@@ -10,6 +10,7 @@ on what type of changes require integrations tests and how you should write inte
   - [New L2 Constructs](#new-l2-constructs)
   - [Existing L2 Constructs](#existing-l2-constructs)
   - [Assertions](#assertions)
+- [Running Integration Tests](#running-integration-tests)
 
 ## What are CDK Integration Tests
 
@@ -223,3 +224,40 @@ to deploy the Lambda Function _and_ then rerun the assertions to ensure that the
 
 ### Assertions
 ...Coming soon...
+
+## Running Integration Tests
+
+Most of the time you will only need to run integration tests for an individual module (i.e. `aws-lambda`). Other times you may need to run tests across multiple modules.
+In this case I would recommend running from the root directory like below.
+
+_Run snapshot tests only_
+```bash
+yarn integ-runner --directory packages/@aws-cdk
+```
+
+_Run snapshot tests and then re-run integration tests for failed snapshots_
+```bash
+yarn integ-runner --directory packages/@aws-cdk --update-on-failed
+```
+
+One benefit of running from the root directory like this is that it will only collect tests from "built" modules. If you have built the entire
+repo it will run all integration tests, but if you have only built a couple modules it will only run tests from those.
+
+### Running large numbers of Tests
+
+If you need to re-run a large number of tests you can run them in parallel like this.
+
+```bash
+yarn integ-runner --directory packages/@aws-cdk --update-on-failed \
+  --parallel-regions us-east-1 \
+  --parallel-regions us-east-2 \
+  --parallel-regions us-west-2 \
+  --parallel-regions eu-west-1 \
+  --profiles profile1 \
+  --profiles profile2 \
+  --profiles profile3 \
+  --verbose
+```
+
+When using both `--parallel-regions` and `--profiles` it will execute (regions*profiles) tests in parallel (in this example 12)
+If you want to execute more than 16 tests in parallel you can pass a higher value to `--max-workers`.

packages/@aws-cdk/cloud-assembly-schema/schema/cloud-assembly.version.json

@@ -1 +1 @@
-{"version":"17.0.0"}
+{"version":"17.0.0"}

packages/@aws-cdk/cloud-assembly-schema/schema/integ.schema.json

@@ -471,4 +471,4 @@
         }
     },
     "$schema": "http://json-schema.org/draft-07/schema#"
-}
+}

packages/@aws-cdk/integ-runner/README.md

@@ -44,9 +44,7 @@ to be a self contained CDK app. The runner will execute the following for each f
 - `--clean` (default=`true`)
   Destroy stacks after deploy (use `--no-clean` for debugging)
 - `--verbose` (default=`false`)
-  verbose logging
-- `--parallel` (default=`true`)
-  Run tests in parallel across default regions
+  verbose logging, including integration test metrics
 - `--parallel-regions` (default=`us-east-1`,`us-east-2`, `us-west-2`)
   List of regions to run tests in. If this is provided then all tests will
   be run in parallel across these regions
@@ -66,11 +64,18 @@ to be a self contained CDK app. The runner will execute the following for each f
 Example:
 
 ```bash
-integ-runner --update --parallel --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./
+integ-runner --update-on-failed --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./
 ```
 
 This will search for integration tests recursively from the current directory and then execute them in parallel across `us-east-1`, `us-east-2`, & `us-west-2`.
 
+If you are providing a list of tests to execute, either as CLI arguments or from a file, the name of the test needs to be relative to the `directory`.
+For example, if there is a test `aws-iam/test/integ.policy.js` and the current working directory is `aws-iam` you would provide `integ.policy.js`
+
+```bash
+yarn integ integ.policy.js
+```
+
 ### Common Workflow
 
 A common workflow to use when running integration tests is to first run the integration tests to see if there are any snapshot differences.

packages/@aws-cdk/integ-runner/THIRD_PARTY_LICENSES

@@ -3,7 +3,7 @@ import * as path from 'path';
 import * as chalk from 'chalk';
 import * as workerpool from 'workerpool';
 import * as logger from './logger';
-import { IntegrationTests, IntegTestConfig } from './runner/integ-tests';
+import { IntegrationTests, IntegTestConfig } from './runner/integration-tests';
 import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers';
 
 // https://github.com/yargs/yargs/issues/1929
@@ -17,13 +17,12 @@ async function main() {
     .usage('Usage: integ-runner [TEST...]')
     .option('list', { type: 'boolean', default: false, desc: 'List tests instead of running them' })
     .option('clean', { type: 'boolean', default: true, desc: 'Skips stack clean up after test is completed (use --no-clean to negate)' })
-    .option('verbose', { type: 'boolean', default: false, alias: 'v', desc: 'Verbose logs' })
+    .option('verbose', { type: 'boolean', default: false, alias: 'v', desc: 'Verbose logs and metrics on integration tests durations' })
     .option('dry-run', { type: 'boolean', default: false, desc: 'do not actually deploy the stack. just update the snapshot (not recommended!)' })
     .option('update-on-failed', { type: 'boolean', default: false, desc: 'rerun integration tests and update snapshots for failed tests.' })
     .option('force', { type: 'boolean', default: false, desc: 'Rerun all integration tests even if tests are passing' })
-    .option('parallel', { type: 'boolean', default: false, desc: 'run integration tests in parallel' })
-    .option('parallel-regions', { type: 'array', desc: 'if --parallel is used then these regions are used to run tests in parallel', nargs: 1, default: [] })
-    .options('directory', { type: 'string', default: 'test', desc: 'starting directory to discover integration tests' })
+    .option('parallel-regions', { type: 'array', desc: 'Tests are run in parallel across these regions. To prevent tests from running in parallel, provide only a single region', nargs: 1, default: [] })
+    .options('directory', { type: 'string', default: 'test', desc: 'starting directory to discover integration tests. Tests will be discovered recursively from this directory' })
     .options('profiles', { type: 'array', desc: 'list of AWS profiles to use. Tests will be run in parallel across each profile+regions', nargs: 1, default: [] })
     .options('max-workers', { type: 'number', desc: 'The max number of workerpool workers to use when running integration tests in parallel', default: 16 })
     .options('exclude', { type: 'boolean', desc: 'All tests should be run, except for the list of tests provided', default: false })
@@ -61,11 +60,11 @@ async function main() {
     if (argv._.length > 0 && fromFile) {
       throw new Error('A list of tests cannot be provided if "--from-file" is provided');
     } else if (argv._.length === 0 && !fromFile) {
-      testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromCliArgs()));
+      testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs()));
     } else if (fromFile) {
-      testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromFile(fromFile)));
+      testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromFile(path.resolve(fromFile))));
     } else {
-      testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromCliArgs(argv._.map((x: any) => x.toString()), exclude)));
+      testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs(argv._.map((x: any) => x.toString()), exclude)));
     }
 
     // always run snapshot tests, but if '--force' is passed then
@@ -93,7 +92,7 @@ async function main() {
         clean: argv.clean,
         dryRun: argv['dry-run'],
         verbose: argv.verbose,
-        updateWorkflow: !argv['disable-update-workflow'],
+        updateWorkflow: !!argv['disable-update-workflow'],
       });
 
       if (argv.clean === false) {

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -1,2 +1,5 @@
-export * from './runners';
-export * from './integ-tests';
+export * from './runner-base';
+export * from './integ-test-suite';
+export * from './integ-test-runner';
+export * from './snapshot-test-runner';
+export * from './integration-tests';