Skip to content

Commit 9e220c8

Browse files
rix0rrrmergify[bot]
rix0rrr
and
mergify[bot]
authored
chore: add integ test that validates full proxy traversal (#33092)
Add a CLI integ test to validate that `cdk deploy` works in a fully network-isolated environment, with only a proxy to go through. This validates that no parts of the CLI setup ignore the proxy configuration, which would otherwise be hard to test. We achieve the network isolation by running the code inside a Docker container where we use `iptables` to drop all network traffic that doesn't go through the Docker host, where we run a proxy. I temporarily bumped the `tsconfig` `target` to try out the `using` syntax (didn't work out with Jest), but that caused some compiler errors around class member initialization that I fixed as well. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
1 parent e4703c1 commit 9e220c8

File tree

7 files changed

+228
-56
lines changed

7 files changed

+228
-56
lines changed

‎packages/@aws-cdk-testing/cli-integ/lib/package-sources/release-source.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@ import { shell, rimraf, addToShellPath } from '../shell';
77

88
export class ReleasePackageSourceSetup implements IPackageSourceSetup {
99
readonly name = 'release';
10-
readonly description = `release @ ${this.version}`;
10+
readonly description: string;
1111

1212
private tempDir?: string;
1313

1414
constructor(private readonly version: string, private readonly frameworkVersion?: string) {
15+
this.description = `release @ ${this.version}`;
1516
}
1617

1718
public async prepare(): Promise<void> {

‎packages/@aws-cdk-testing/cli-integ/lib/package-sources/repo-source.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ import { shell, addToShellPath } from '../shell';
77

88
export class RepoPackageSourceSetup implements IPackageSourceSetup {
99
readonly name = 'repo';
10-
readonly description = `repo(${this.repoRoot})`;
10+
readonly description: string;
1111

1212
constructor(private readonly repoRoot: string) {
13+
this.description = `repo(${this.repoRoot})`;
1314
}
1415

1516
public async prepare(): Promise<void> {

‎packages/@aws-cdk-testing/cli-integ/lib/proxy.ts

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import { promises as fs } from 'fs';
2+
import * as querystring from 'node:querystring';
3+
import * as os from 'os';
4+
import * as path from 'path';
5+
import * as mockttp from 'mockttp';
6+
import { CompletedRequest } from 'mockttp';
7+
8+
export async function startProxyServer(certDirRoot?: string): Promise<ProxyServer> {
9+
const certDir = await fs.mkdtemp(path.join(certDirRoot ?? os.tmpdir(), 'cdk-'));
10+
const certPath = path.join(certDir, 'cert.pem');
11+
const keyPath = path.join(certDir, 'key.pem');
12+
13+
// Set up key and certificate
14+
const { key, cert } = await mockttp.generateCACertificate();
15+
await fs.writeFile(keyPath, key);
16+
await fs.writeFile(certPath, cert);
17+
18+
const server = mockttp.getLocal({
19+
https: { keyPath: keyPath, certPath: certPath },
20+
});
21+
22+
// We don't need to modify any request, so the proxy
23+
// passes through all requests to the target host.
24+
const endpoint = await server
25+
.forAnyRequest()
26+
.thenPassThrough();
27+
28+
server.enableDebug();
29+
await server.start();
30+
31+
return {
32+
certPath,
33+
keyPath,
34+
server,
35+
url: server.url,
36+
port: server.port,
37+
getSeenRequests: () => endpoint.getSeenRequests(),
38+
async stop() {
39+
await server.stop();
40+
await fs.rm(certDir, { recursive: true, force: true });
41+
},
42+
};
43+
}
44+
45+
export interface ProxyServer {
46+
readonly certPath: string;
47+
readonly keyPath: string;
48+
readonly server: mockttp.Mockttp;
49+
readonly url: string;
50+
readonly port: number;
51+
52+
getSeenRequests(): Promise<CompletedRequest[]>;
53+
stop(): Promise<void>;
54+
}
55+
56+
export function awsActionsFromRequests(requests: CompletedRequest[]): string[] {
57+
return [...new Set(requests
58+
.map(req => req.body.buffer.toString('utf-8'))
59+
.map(body => querystring.decode(body))
60+
.map(x => x.Action as string)
61+
.filter(action => action != null))];
62+
}

‎packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ export interface CdkGarbageCollectionCommandOptions {
317317
}
318318

319319
export class TestFixture extends ShellHelper {
320-
public readonly qualifier = this.randomString.slice(0, 10);
320+
public readonly qualifier: string;
321321
private readonly bucketsToDelete = new Array<string>();
322322
public readonly packages: IPackageSource;
323323

@@ -330,6 +330,7 @@ export class TestFixture extends ShellHelper {
330330

331331
super(integTestDir, output);
332332

333+
this.qualifier = this.randomString.slice(0, 10);
333334
this.packages = packageSourceInSubprocess();
334335
}
335336

@@ -338,16 +339,22 @@ export class TestFixture extends ShellHelper {
338339
}
339340

340341
public async cdkDeploy(stackNames: string | string[], options: CdkCliOptions = {}, skipStackRename?: boolean) {
341-
stackNames = typeof stackNames === 'string' ? [stackNames] : stackNames;
342+
return this.cdk(this.cdkDeployCommandLine(stackNames, options, skipStackRename));
343+
}
342344

345+
public cdkDeployCommandLine(stackNames: string | string[], options: CdkCliOptions = {}, skipStackRename?: boolean) {
346+
stackNames = typeof stackNames === 'string' ? [stackNames] : stackNames;
343347
const neverRequireApproval = options.neverRequireApproval ?? true;
344348

345-
return this.cdk(['deploy',
349+
return [
350+
'deploy',
346351
...(neverRequireApproval ? ['--require-approval=never'] : []), // Default to no approval in an unattended test
347352
...(options.options ?? []),
353+
...(options.verbose ? ['-v'] : []),
348354
// use events because bar renders bad in tests
349355
'--progress', 'events',
350-
...(skipStackRename ? stackNames : this.fullStackName(stackNames))], options);
356+
...(skipStackRename ? stackNames : this.fullStackName(stackNames)),
357+
];
351358
}
352359

353360
public async cdkSynth(options: CdkCliOptions = {}) {
@@ -490,15 +497,24 @@ export class TestFixture extends ShellHelper {
490497
return this.shell(['cdk', ...(verbose ? ['-v'] : []), ...args], {
491498
...options,
492499
modEnv: {
493-
AWS_REGION: this.aws.region,
494-
AWS_DEFAULT_REGION: this.aws.region,
495-
STACK_NAME_PREFIX: this.stackNamePrefix,
496-
PACKAGE_LAYOUT_VERSION: this.packages.majorVersion(),
500+
...this.cdkShellEnv(),
497501
...options.modEnv,
498502
},
499503
});
500504
}
501505

506+
/**
507+
* Return the environment variables with which to execute CDK
508+
*/
509+
public cdkShellEnv() {
510+
return {
511+
AWS_REGION: this.aws.region,
512+
AWS_DEFAULT_REGION: this.aws.region,
513+
STACK_NAME_PREFIX: this.stackNamePrefix,
514+
PACKAGE_LAYOUT_VERSION: this.packages.majorVersion(),
515+
};
516+
}
517+
502518
public template(stackName: string): any {
503519
const fullStackName = this.fullStackName(stackName);
504520
const templatePath = path.join(this.integTestDir, 'cdk.out', `${fullStackName}.template.json`);

‎packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts

Lines changed: 12 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import { existsSync, promises as fs } from 'fs';
2-
import * as querystring from 'node:querystring';
32
import * as os from 'os';
43
import * as path from 'path';
54
import {
@@ -23,8 +22,6 @@ import { InvokeCommand } from '@aws-sdk/client-lambda';
2322
import { PutObjectLockConfigurationCommand } from '@aws-sdk/client-s3';
2423
import { CreateTopicCommand, DeleteTopicCommand } from '@aws-sdk/client-sns';
2524
import { AssumeRoleCommand, GetCallerIdentityCommand } from '@aws-sdk/client-sts';
26-
import * as mockttp from 'mockttp';
27-
import { CompletedRequest } from 'mockttp';
2825
import {
2926
cloneDirectory,
3027
integTest,
@@ -41,6 +38,7 @@ import {
4138
withSamIntegrationFixture,
4239
withSpecificFixture,
4340
} from '../../lib';
41+
import { awsActionsFromRequests, startProxyServer } from '../../lib/proxy';
4442

4543
jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
4644

@@ -2876,60 +2874,29 @@ integTest('cdk notices are displayed correctly', withDefaultFixture(async (fixtu
28762874

28772875
integTest('requests go through a proxy when configured',
28782876
withDefaultFixture(async (fixture) => {
2879-
// Set up key and certificate
2880-
const { key, cert } = await mockttp.generateCACertificate();
2881-
const certDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-'));
2882-
const certPath = path.join(certDir, 'cert.pem');
2883-
const keyPath = path.join(certDir, 'key.pem');
2884-
await fs.writeFile(keyPath, key);
2885-
await fs.writeFile(certPath, cert);
2886-
2887-
const proxyServer = mockttp.getLocal({
2888-
https: { keyPath, certPath },
2889-
});
2890-
2891-
// We don't need to modify any request, so the proxy
2892-
// passes through all requests to the target host.
2893-
const endpoint = await proxyServer
2894-
.forAnyRequest()
2895-
.thenPassThrough();
2896-
2897-
proxyServer.enableDebug();
2898-
await proxyServer.start();
2899-
2900-
// The proxy is now ready to intercept requests
2901-
2877+
const proxyServer = await startProxyServer();
29022878
try {
29032879
await fixture.cdkDeploy('test-2', {
29042880
captureStderr: true,
29052881
options: [
29062882
'--proxy', proxyServer.url,
2907-
'--ca-bundle-path', certPath,
2883+
'--ca-bundle-path', proxyServer.certPath,
29082884
],
29092885
modEnv: {
29102886
CDK_HOME: fixture.integTestDir,
29112887
},
29122888
});
2913-
} finally {
2914-
await fs.rm(certDir, { recursive: true, force: true });
2915-
await proxyServer.stop();
2916-
}
29172889

2918-
const requests = await endpoint.getSeenRequests();
2890+
const requests = await proxyServer.getSeenRequests();
29192891

2920-
expect(requests.map(req => req.url))
2921-
.toContain('https://cli.cdk.dev-tools.aws.dev/notices.json');
2892+
expect(requests.map(req => req.url))
2893+
.toContain('https://cli.cdk.dev-tools.aws.dev/notices.json');
29222894

2923-
const actionsUsed = actions(requests);
2924-
expect(actionsUsed).toContain('AssumeRole');
2925-
expect(actionsUsed).toContain('CreateChangeSet');
2895+
const actionsUsed = awsActionsFromRequests(requests);
2896+
expect(actionsUsed).toContain('AssumeRole');
2897+
expect(actionsUsed).toContain('CreateChangeSet');
2898+
} finally {
2899+
await proxyServer.stop();
2900+
}
29262901
}),
29272902
);
2928-
2929-
function actions(requests: CompletedRequest[]): string[] {
2930-
return [...new Set(requests
2931-
.map(req => req.body.buffer.toString('utf-8'))
2932-
.map(body => querystring.decode(body))
2933-
.map(x => x.Action as string)
2934-
.filter(action => action != null))];
2935-
}

‎packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/proxy.integtest.ts

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import { promises as fs } from 'fs';
2+
import * as path from 'path';
3+
import { integTest } from '../../lib/integ-test';
4+
import { startProxyServer } from '../../lib/proxy';
5+
import { TestFixture, withDefaultFixture } from '../../lib/with-cdk-app';
6+
7+
const docker = process.env.CDK_DOCKER ?? 'docker';
8+
9+
integTest(
10+
'deploy in isolated container',
11+
withDefaultFixture(async (fixture) => {
12+
// Find the 'cdk' command and make sure it is mounted into the container
13+
const cdkFullpath = (await fixture.shell(['which', 'cdk'])).trim();
14+
const cdkTop = topLevelDirectory(cdkFullpath);
15+
16+
// Run a 'cdk deploy' inside the container
17+
const commands = [
18+
`env ${renderEnv(fixture.cdkShellEnv())} ${cdkFullpath} ${fixture.cdkDeployCommandLine('test-2', { verbose: true }).join(' ')}`,
19+
];
20+
21+
await runInIsolatedContainer(fixture, [cdkTop], commands);
22+
}),
23+
);
24+
25+
async function runInIsolatedContainer(fixture: TestFixture, pathsToMount: string[], testCommands: string[]) {
26+
pathsToMount.push(
27+
`${process.env.HOME}`,
28+
fixture.integTestDir,
29+
);
30+
31+
const proxy = await startProxyServer(fixture.integTestDir);
32+
try {
33+
const proxyPort = proxy.port;
34+
35+
const setupCommands = [
36+
'apt-get update -qq',
37+
'apt-get install -qqy nodejs > /dev/null',
38+
...isolatedDockerCommands(proxyPort, proxy.certPath),
39+
];
40+
41+
const scriptName = path.join(fixture.integTestDir, 'script.sh');
42+
43+
// Write a script file
44+
await fs.writeFile(scriptName, [
45+
'#!/bin/bash',
46+
'set -x',
47+
'set -eu',
48+
...setupCommands,
49+
...testCommands,
50+
].join('\n'), 'utf-8');
51+
52+
await fs.chmod(scriptName, 0o755);
53+
54+
// Run commands in a Docker shell
55+
await fixture.shell([
56+
docker, 'run', '--net=bridge', '--rm',
57+
...pathsToMount.flatMap(p => ['-v', `${p}:${p}`]),
58+
...['HOME', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'].flatMap(e => ['-e', e]),
59+
'-w', fixture.integTestDir,
60+
'--cap-add=NET_ADMIN',
61+
'ubuntu:latest',
62+
`${scriptName}`,
63+
], {
64+
stdio: 'inherit',
65+
});
66+
} finally {
67+
await proxy.stop();
68+
}
69+
}
70+
71+
function topLevelDirectory(dir: string) {
72+
while (true) {
73+
let parent = path.dirname(dir);
74+
if (parent === '/') {
75+
return dir;
76+
}
77+
dir = parent;
78+
}
79+
}
80+
81+
/**
82+
* Return the commands necessary to isolate the inside of the container from the internet,
83+
* except by going through the proxy
84+
*/
85+
function isolatedDockerCommands(proxyPort: number, caBundlePath: string) {
86+
return [
87+
'echo Working...',
88+
'apt-get install -qqy curl net-tools iputils-ping dnsutils iptables > /dev/null',
89+
'',
90+
'gateway=$(dig +short host.docker.internal)',
91+
'',
92+
'# Some iptables manipulation; there might be unnecessary commands in here, not an expert',
93+
'iptables -F',
94+
'iptables -X',
95+
'iptables -P INPUT DROP',
96+
'iptables -P OUTPUT DROP',
97+
'iptables -P FORWARD DROP',
98+
'iptables -A INPUT -i lo -j ACCEPT',
99+
'iptables -A OUTPUT -o lo -j ACCEPT',
100+
'iptables -A OUTPUT -d $gateway -j ACCEPT',
101+
'iptables -A INPUT -s $gateway -j ACCEPT',
102+
'',
103+
'',
104+
`if [[ ! -f ${caBundlePath} ]]; then`,
105+
` echo "Could not find ${caBundlePath}, this will probably not go well. Exiting." >&2`,
106+
' exit 1',
107+
'fi',
108+
'',
109+
'# Configure a bunch of tools to work with the proxy',
110+
'echo "+-------------------------------------------------------------------------------------+"',
111+
'echo "| Direct network traffic has been blocked, everything must go through the proxy. |"',
112+
'echo "+-------------------------------------------------------------------------------------+"',
113+
`export HTTP_PROXY=http://$gateway:${proxyPort}/`,
114+
`export HTTPS_PROXY=http://$gateway:${proxyPort}/`,
115+
`export NODE_EXTRA_CA_CERTS=${caBundlePath}`,
116+
`export AWS_CA_BUNDLE=${caBundlePath}`,
117+
`export SSL_CERT_FILE=${caBundlePath}`,
118+
'echo "Acquire::http::proxy \"$HTTP_PROXY\";" >> /etc/apt/apt.conf.d/95proxies',
119+
'echo "Acquire::https::proxy \"$HTTPS_PROXY\";" >> /etc/apt/apt.conf.d/95proxies',
120+
];
121+
}
122+
123+
function renderEnv(env: Record<string, string>) {
124+
return Object.entries(env).map(([k, v]) => `${k}='${v}'`).join(' ');
125+
}

‎packages/@aws-cdk-testing/cli-integ/tsconfig.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"compilerOptions": {
33
"target": "ES2020",
44
"module": "commonjs",
5-
"lib": ["es2019", "es2020", "dom"],
5+
"lib": ["ES2020", "dom"],
66
"strict": true,
77
"alwaysStrict": true,
88
"declaration": true,

0 commit comments

Comments
 (0)