chore(cli-integ): add per-test timeouts (#24504)

We couldn't use the jest timeout feature for our integration tests, and ended up running without timeouts.

This made it EXTREMELY hard to debug an issue where the tests ended up not finishing, and not producing any output to indicate why.

Re-add the ability to have timeouts on test, and set a timeout of 10 minutes on all of them.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: rix0rrr

packages/@aws-cdk-testing/cli-integ/lib/resource-pool.ts

@@ -87,15 +87,14 @@ export class ResourcePool<A extends string=string> {
 
   private makeLease(value: A): ILease<A> {
     let disposed = false;
-    process.stderr.write(`Lease acquired by ${process.pid}: ${value}`);
     return {
       value,
       dispose: async () => {
         if (disposed) {
           throw new Error('Calling dispose() on an already-disposed lease.');
         }
         disposed = true;
-        return this.returnValue(value).finally(() => process.stderr.write(`Lease returned by ${process.pid}: ${value}`));
+        return this.returnValue(value);
       },
     };
   }

packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts

@@ -9,6 +9,9 @@ import { packageSourceInSubprocess } from './package-sources/subprocess';
 import { RESOURCES_DIR } from './resources';
 import { shell, ShellOptions, ShellHelper, rimraf } from './shell';
 import { AwsContext, withAws } from './with-aws';
+import { withTimeout } from './with-timeout';
+
+export const DEFAULT_TEST_TIMEOUT_S = 10 * 60;
 
 /**
  * Higher order function to execute a block with a CDK app fixture
@@ -135,7 +138,7 @@ export function withMonolithicCfnIncludeCdkApp<A extends TestContext>(block: (co
  * test declaration but centralizing it is going to make it convenient to modify in the future.
  */
 export function withDefaultFixture(block: (context: TestFixture) => Promise<void>) {
-  return withAws(withCdkApp(block));
+  return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withCdkApp(block)));
 }
 
 export interface DisableBootstrapContext {

packages/@aws-cdk-testing/cli-integ/lib/with-sam.ts

@@ -7,7 +7,8 @@ import { TestContext } from './integ-test';
 import { RESOURCES_DIR } from './resources';
 import { ShellOptions, rimraf } from './shell';
 import { AwsContext, withAws } from './with-aws';
-import { cloneDirectory, installNpmPackages, TestFixture } from './with-cdk-app';
+import { cloneDirectory, installNpmPackages, TestFixture, DEFAULT_TEST_TIMEOUT_S } from './with-cdk-app';
+import { withTimeout } from './with-timeout';
 
 
 export interface ActionOutput {
@@ -113,7 +114,7 @@ function errorCausedByGoPkg(error: string) {
  * SAM Integration test fixture for CDK - SAM integration test cases
  */
 export function withSamIntegrationFixture(block: (context: SamIntegrationTestFixture) => Promise<void>) {
-  return withAws(withSamIntegrationCdkApp(block));
+  return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withSamIntegrationCdkApp(block)));
 }
 
 export class SamIntegrationTestFixture extends TestFixture {

packages/@aws-cdk-testing/cli-integ/lib/with-timeout.ts

@@ -0,0 +1,33 @@
+/**
+ * Run a block with a timeout
+ *
+ * We can't use the jest timeout feature:
+ *
+ * - `jest.concurrent()` does not do any concurrency management. It starts all
+ *   tests at the same time.
+ * - Our tests use locking to make sure only one test is running at a time per
+ *   region.
+ *
+ * The wait time for the locks is included in the jest test timeout. We therefore
+ * need to set it unreasonably high (as long as the last test may need to wait
+ * if all tests are executed using only 1 region, and they effectively execute
+ * sequentially), which makes it not useful to detect stuck tests.
+ *
+ * The `withTimeout()` modifier makes it possible to measure only a specific
+ * block of code. In our case: the effective test code, excluding the wait time.
+ */
+export function withTimeout<A>(seconds: number, block: (x: A) => Promise<void>) {
+  return (x: A) => {
+    const timeOut = new Promise<void>((_ok, ko) => {
+      const timerHandle = setTimeout(
+        () => ko(new Error(`Timeout: test took more than ${seconds}s to complete`)),
+        seconds * 1000);
+      timerHandle.unref();
+    });
+
+    return Promise.race([
+      block(x),
+      timeOut,
+    ]);
+  };
+}

packages/@aws-cdk-testing/cli-integ/test/resource-pool.test.ts

@@ -32,7 +32,7 @@ test('double dispose throws', async () => {
   const lease = await pool.take();
 
   await lease.dispose();
-  expect(() => lease.dispose()).toThrow();
+  await expect(() => lease.dispose()).rejects.toThrow();
 });
 
 test('somewhat balance', async () => {