chore: add integ test that validates full proxy traversal (#33140)

(Re-roll of #33092)

Add a CLI integ test to validate that `cdk deploy` works in a fully network-isolated environment, with only a proxy to go through.

This validates that no parts of the CLI setup ignore the proxy configuration, which would otherwise be hard to test.

We achieve the network isolation by running the code inside a Docker container where we use `iptables` to drop all network traffic that doesn't go through the Docker host, where we run a proxy.

I temporarily bumped the `tsconfig` `target` to try out the `using` syntax (didn't work out with Jest), but that caused some compiler errors around class member initialization that I fixed as well.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: rix0rrr

packages/@aws-cdk-testing/cli-integ/lib/aws.ts

@@ -19,11 +19,11 @@ import {
 import { SNSClient } from '@aws-sdk/client-sns';
 import { SSOClient } from '@aws-sdk/client-sso';
 import { STSClient, GetCallerIdentityCommand } from '@aws-sdk/client-sts';
-import { fromIni } from '@aws-sdk/credential-providers';
+import { fromIni, fromNodeProviderChain } from '@aws-sdk/credential-providers';
 import type { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@smithy/types';
 import { ConfiguredRetryStrategy } from '@smithy/util-retry';
 interface ClientConfig {
-  readonly credentials?: AwsCredentialIdentityProvider | AwsCredentialIdentity;
+  readonly credentials: AwsCredentialIdentityProvider | AwsCredentialIdentity;
   readonly region: string;
   readonly retryStrategy: ConfiguredRetryStrategy;
 }
@@ -80,6 +80,17 @@ export class AwsClients {
     return (await stsClient.send(new GetCallerIdentityCommand({}))).Account!;
   }
 
+  /**
+   * Resolve the current identity or identity provider to credentials
+   */
+  public async credentials() {
+    const x = this.config.credentials;
+    if (isAwsCredentialIdentity(x)) {
+      return x;
+    }
+    return x();
+  }
+
   public async deleteStacks(...stackNames: string[]) {
     if (stackNames.length === 0) {
       return;
@@ -251,7 +262,7 @@ export async function sleep(ms: number) {
   return new Promise((ok) => setTimeout(ok, ms));
 }
 
-function chainableCredentials(region: string): AwsCredentialIdentityProvider | undefined {
+function chainableCredentials(region: string): AwsCredentialIdentityProvider {
   if ((process.env.CODEBUILD_BUILD_ARN || process.env.GITHUB_RUN_ID) && process.env.AWS_PROFILE) {
     // in codebuild we must assume the role that the cdk uses
     // otherwise credentials will just be picked up by the normal sdk
@@ -261,5 +272,10 @@ function chainableCredentials(region: string): AwsCredentialIdentityProvider | u
     });
   }
 
-  return undefined;
+  // Otherwise just get what's default
+  return fromNodeProviderChain({ clientConfig: { region } });
+}
+
+function isAwsCredentialIdentity(x: any): x is AwsCredentialIdentity {
+  return Boolean(x && typeof x === 'object' && x.accessKeyId);
 }

packages/@aws-cdk-testing/cli-integ/lib/cli/run-suite.ts

@@ -121,6 +121,7 @@ async function main() {
 
   try {
     await jest.run([
+      '--randomize',
       ...args.runInBand ? ['-i'] : [],
       ...args.test ? ['-t', args.test] : [],
       ...args.verbose ? ['--verbose'] : [],

packages/@aws-cdk-testing/cli-integ/lib/package-sources/release-source.ts

@@ -7,11 +7,12 @@ import { shell, rimraf, addToShellPath } from '../shell';
 
 export class ReleasePackageSourceSetup implements IPackageSourceSetup {
   readonly name = 'release';
-  readonly description = `release @ ${this.version}`;
+  readonly description: string;
 
   private tempDir?: string;
 
   constructor(private readonly version: string, private readonly frameworkVersion?: string) {
+    this.description = `release @ ${this.version}`;
   }
 
   public async prepare(): Promise<void> {

packages/@aws-cdk-testing/cli-integ/lib/package-sources/repo-source.ts

@@ -7,9 +7,10 @@ import { shell, addToShellPath } from '../shell';
 
 export class RepoPackageSourceSetup implements IPackageSourceSetup {
   readonly name = 'repo';
-  readonly description = `repo(${this.repoRoot})`;
+  readonly description: string;
 
   constructor(private readonly repoRoot: string) {
+    this.description = `repo(${this.repoRoot})`;
   }
 
   public async prepare(): Promise<void> {

packages/@aws-cdk-testing/cli-integ/lib/proxy.ts

@@ -0,0 +1,64 @@
+import { promises as fs } from 'fs';
+import * as querystring from 'node:querystring';
+import * as os from 'os';
+import * as path from 'path';
+import * as mockttp from 'mockttp';
+import { CompletedRequest } from 'mockttp';
+
+export async function startProxyServer(certDirRoot?: string): Promise<ProxyServer> {
+  const certDir = await fs.mkdtemp(path.join(certDirRoot ?? os.tmpdir(), 'cdk-'));
+  const certPath = path.join(certDir, 'cert.pem');
+  const keyPath = path.join(certDir, 'key.pem');
+
+  // Set up key and certificate
+  const { key, cert } = await mockttp.generateCACertificate();
+  await fs.writeFile(keyPath, key);
+  await fs.writeFile(certPath, cert);
+
+  const server = mockttp.getLocal({
+    https: { keyPath: keyPath, certPath: certPath },
+  });
+
+  // We don't need to modify any request, so the proxy
+  // passes through all requests to the target host.
+  const endpoint = await server
+    .forAnyRequest()
+    .thenPassThrough();
+
+  const port = 9000 + Math.floor(Math.random() * 10000);
+
+  // server.enableDebug();
+  await server.start(port);
+
+  return {
+    certPath,
+    keyPath,
+    server,
+    url: server.url,
+    port: server.port,
+    getSeenRequests: () => endpoint.getSeenRequests(),
+    async stop() {
+      await server.stop();
+      await fs.rm(certDir, { recursive: true, force: true });
+    },
+  };
+}
+
+export interface ProxyServer {
+  readonly certPath: string;
+  readonly keyPath: string;
+  readonly server: mockttp.Mockttp;
+  readonly url: string;
+  readonly port: number;
+
+  getSeenRequests(): Promise<CompletedRequest[]>;
+  stop(): Promise<void>;
+}
+
+export function awsActionsFromRequests(requests: CompletedRequest[]): string[] {
+  return [...new Set(requests
+    .map(req => req.body.buffer.toString('utf-8'))
+    .map(body => querystring.decode(body))
+    .map(x => x.Action as string)
+    .filter(action => action != null))];
+}

packages/@aws-cdk-testing/cli-integ/lib/shell.ts

@@ -21,9 +21,15 @@ export async function shell(command: string[], options: ShellOptions = {}): Prom
     }
   };
 
-  // Always output the command
-  writeToOutputs(`💻 ${command.join(' ')}\n`);
   const show = options.show ?? 'always';
+  const verbose = Boolean(process.env.VERBOSE);
+
+  if (verbose) {
+    outputs.add(process.stdout);
+  }
+
+  // Always output the command
+  writeToOutputs(`💻 ${command.join(' ')} (show: ${show}, verbose: ${verbose})\n`);
 
   const env = options.env ?? (options.modEnv ? { ...process.env, ...options.modEnv } : process.env);
 

packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts

@@ -321,7 +321,7 @@ export interface CdkGarbageCollectionCommandOptions {
 }
 
 export class TestFixture extends ShellHelper {
-  public readonly qualifier = this.randomString.slice(0, 10);
+  public readonly qualifier: string;
   private readonly bucketsToDelete = new Array<string>();
   public readonly packages: IPackageSource;
 
@@ -333,6 +333,7 @@ export class TestFixture extends ShellHelper {
     public readonly randomString: string) {
     super(integTestDir, output);
 
+    this.qualifier = this.randomString.slice(0, 10);
     this.packages = packageSourceInSubprocess();
   }
 
@@ -341,16 +342,21 @@ export class TestFixture extends ShellHelper {
   }
 
   public async cdkDeploy(stackNames: string | string[], options: CdkCliOptions = {}, skipStackRename?: boolean) {
-    stackNames = typeof stackNames === 'string' ? [stackNames] : stackNames;
+    return this.cdk(this.cdkDeployCommandLine(stackNames, options, skipStackRename), options);
+  }
 
+  public cdkDeployCommandLine(stackNames: string | string[], options: CdkCliOptions = {}, skipStackRename?: boolean) {
+    stackNames = typeof stackNames === 'string' ? [stackNames] : stackNames;
     const neverRequireApproval = options.neverRequireApproval ?? true;
 
-    return this.cdk(['deploy',
+    return [
+      'deploy',
       ...(neverRequireApproval ? ['--require-approval=never'] : []), // Default to no approval in an unattended test
       ...(options.options ?? []),
       // use events because bar renders bad in tests
       '--progress', 'events',
-      ...(skipStackRename ? stackNames : this.fullStackName(stackNames))], options);
+      ...(skipStackRename ? stackNames : this.fullStackName(stackNames)),
+    ];
   }
 
   public async cdkSynth(options: CdkCliOptions = {}) {
@@ -497,6 +503,19 @@ export class TestFixture extends ShellHelper {
 
     await this.packages.makeCliAvailable();
 
+    return this.shell(['cdk', ...(verbose ? ['-v'] : []), ...args], {
+      ...options,
+      modEnv: {
+        ...this.cdkShellEnv(),
+        ...options.modEnv,
+      },
+    });
+  }
+
+  /**
+   * Return the environment variables with which to execute CDK
+   */
+  public cdkShellEnv() {
     // if tests are using an explicit aws identity already (i.e creds)
     // force every cdk command to use the same identity.
     const awsCreds: Record<string, string> = this.aws.identity ? {
@@ -505,19 +524,15 @@ export class TestFixture extends ShellHelper {
       AWS_SESSION_TOKEN: this.aws.identity.sessionToken!,
     } : {};
 
-    return this.shell(['cdk', ...(verbose ? ['-v'] : []), ...args], {
-      ...options,
-      modEnv: {
-        AWS_REGION: this.aws.region,
-        AWS_DEFAULT_REGION: this.aws.region,
-        STACK_NAME_PREFIX: this.stackNamePrefix,
-        PACKAGE_LAYOUT_VERSION: this.packages.majorVersion(),
-        // CI must be unset, because we're trying to capture stdout in a bunch of tests
-        CI: 'false',
-        ...awsCreds,
-        ...options.modEnv,
-      },
-    });
+    return {
+      AWS_REGION: this.aws.region,
+      AWS_DEFAULT_REGION: this.aws.region,
+      STACK_NAME_PREFIX: this.stackNamePrefix,
+      PACKAGE_LAYOUT_VERSION: this.packages.majorVersion(),
+      // In these tests we want to make a distinction between stdout and sterr
+      CI: 'false',
+      ...awsCreds,
+    };
   }
 
   public template(stackName: string): any {

packages/@aws-cdk-testing/cli-integ/lib/with-cli-lib.ts

@@ -131,8 +131,7 @@ __EOS__`], {
         AWS_DEFAULT_REGION: this.aws.region,
         STACK_NAME_PREFIX: this.stackNamePrefix,
         PACKAGE_LAYOUT_VERSION: this.packages.majorVersion(),
-        // Unset CI because we need to distinguish stdout/stderr and this variable
-        // makes everything go to stdout
+        // In these tests we want to make a distinction between stdout and sterr
         CI: 'false',
         ...options.modEnv,
       },

packages/@aws-cdk-testing/cli-integ/resources/integ.jest.config.js

@@ -14,7 +14,10 @@ module.exports = {
   moduleFileExtensions: ["js"],
 
   testEnvironment: "node",
-  testTimeout: 300000,
+
+  // Because of the way Jest concurrency works, this timeout includes waiting
+  // for the lock. Which is almost never what we actually care about. Set it high.
+  testTimeout: 600000,
 
   // Affects test.concurrent(), these are self-limiting anyway
   maxConcurrency: 10,

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts

@@ -1,5 +1,4 @@
 import { existsSync, promises as fs } from 'fs';
-import * as querystring from 'node:querystring';
 import * as os from 'os';
 import * as path from 'path';
 import {
@@ -23,8 +22,6 @@ import { InvokeCommand } from '@aws-sdk/client-lambda';
 import { PutObjectLockConfigurationCommand } from '@aws-sdk/client-s3';
 import { CreateTopicCommand, DeleteTopicCommand } from '@aws-sdk/client-sns';
 import { AssumeRoleCommand, GetCallerIdentityCommand } from '@aws-sdk/client-sts';
-import * as mockttp from 'mockttp';
-import { CompletedRequest } from 'mockttp';
 import {
   cloneDirectory,
   integTest,
@@ -41,6 +38,7 @@ import {
   withSamIntegrationFixture,
   withSpecificFixture,
 } from '../../lib';
+import { awsActionsFromRequests, startProxyServer } from '../../lib/proxy';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
@@ -2873,60 +2871,32 @@ integTest('cdk notices are displayed correctly', withDefaultFixture(async (fixtu
 
 integTest('requests go through a proxy when configured',
   withDefaultFixture(async (fixture) => {
-    // Set up key and certificate
-    const { key, cert } = await mockttp.generateCACertificate();
-    const certDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-'));
-    const certPath = path.join(certDir, 'cert.pem');
-    const keyPath = path.join(certDir, 'key.pem');
-    await fs.writeFile(keyPath, key);
-    await fs.writeFile(certPath, cert);
-
-    const proxyServer = mockttp.getLocal({
-      https: { keyPath, certPath },
-    });
-
-    // We don't need to modify any request, so the proxy
-    // passes through all requests to the target host.
-    const endpoint = await proxyServer
-      .forAnyRequest()
-      .thenPassThrough();
-
-    proxyServer.enableDebug();
-    await proxyServer.start();
-
-    // The proxy is now ready to intercept requests
-
+    const proxyServer = await startProxyServer();
     try {
+      // Delete notices cache if it exists
+      await fs.rm(path.join(process.env.HOME ?? os.userInfo().homedir, '.cdk/cache/notices.json'), { force: true });
+
       await fixture.cdkDeploy('test-2', {
         captureStderr: true,
         options: [
           '--proxy', proxyServer.url,
-          '--ca-bundle-path', certPath,
+          '--ca-bundle-path', proxyServer.certPath,
         ],
         modEnv: {
           CDK_HOME: fixture.integTestDir,
         },
       });
-    } finally {
-      await fs.rm(certDir, { recursive: true, force: true });
-      await proxyServer.stop();
-    }
 
-    const requests = await endpoint.getSeenRequests();
+      const requests = await proxyServer.getSeenRequests();
 
-    expect(requests.map(req => req.url))
-      .toContain('https://cli.cdk.dev-tools.aws.dev/notices.json');
+      expect(requests.map(req => req.url))
+        .toContain('https://cli.cdk.dev-tools.aws.dev/notices.json');
 
-    const actionsUsed = actions(requests);
-    expect(actionsUsed).toContain('AssumeRole');
-    expect(actionsUsed).toContain('CreateChangeSet');
+      const actionsUsed = awsActionsFromRequests(requests);
+      expect(actionsUsed).toContain('AssumeRole');
+      expect(actionsUsed).toContain('CreateChangeSet');
+    } finally {
+      await proxyServer.stop();
+    }
   }),
 );
-
-function actions(requests: CompletedRequest[]): string[] {
-  return [...new Set(requests
-    .map(req => req.body.buffer.toString('utf-8'))
-    .map(body => querystring.decode(body))
-    .map(x => x.Action as string)
-    .filter(action => action != null))];
-}