Skip to content

Commit 78b34ac

Browse files
mrgrainmrgrain
authored
fix(cli-lib): deploy fails with "no such file or directory, open 'node_modules/@aws-cdk/integ-runner/lib/workers/db.json.gz'" (#28199)
After #27813 the `deploy` action was broken with the above error. This is effectively the same as #27983 . To ensure these kind of issues are not slipping through again, the PR is adding a basic testing harness for `cli-lib` to `@aws-cdk-testing/cli-integtests`. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 116b933 commit 78b34ac

File tree

9 files changed

+239
-3
lines changed

9 files changed

+239
-3
lines changed

‎packages/@aws-cdk-testing/cli-integ/lib/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export * from './corking';
33
export * from './integ-test';
44
export * from './memoize';
55
export * from './resource-pool';
6+
export * from './with-cli-lib';
67
export * from './with-sam';
78
export * from './shell';
89
export * from './with-aws';

‎packages/@aws-cdk-testing/cli-integ/lib/with-cli-lib.ts

+134
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import * as os from 'os';
2+
import * as path from 'path';
3+
import { TestContext } from './integ-test';
4+
import { RESOURCES_DIR } from './resources';
5+
import { AwsContext, withAws } from './with-aws';
6+
import { cloneDirectory, installNpmPackages, TestFixture, DEFAULT_TEST_TIMEOUT_S, CdkCliOptions } from './with-cdk-app';
7+
import { withTimeout } from './with-timeout';
8+
9+
/**
10+
* Higher order function to execute a block with a CliLib Integration CDK app fixture
11+
*/
12+
export function withCliLibIntegrationCdkApp<A extends TestContext & AwsContext>(block: (context: CliLibIntegrationTestFixture) => Promise<void>) {
13+
return async (context: A) => {
14+
const randy = context.randomString;
15+
const stackNamePrefix = `cdktest-${randy}`;
16+
const integTestDir = path.join(os.tmpdir(), `cdk-integ-${randy}`);
17+
18+
context.log(` Stack prefix: ${stackNamePrefix}\n`);
19+
context.log(` Test directory: ${integTestDir}\n`);
20+
context.log(` Region: ${context.aws.region}\n`);
21+
22+
await cloneDirectory(path.join(RESOURCES_DIR, 'cdk-apps', 'simple-app'), integTestDir, context.output);
23+
const fixture = new CliLibIntegrationTestFixture(
24+
integTestDir,
25+
stackNamePrefix,
26+
context.output,
27+
context.aws,
28+
context.randomString);
29+
30+
let success = true;
31+
try {
32+
const installationVersion = fixture.packages.requestedFrameworkVersion();
33+
34+
if (fixture.packages.majorVersion() === '1') {
35+
throw new Error('This test suite is only compatible with AWS CDK v2');
36+
}
37+
38+
const alphaInstallationVersion = fixture.packages.requestedAlphaVersion();
39+
await installNpmPackages(fixture, {
40+
'aws-cdk-lib': installationVersion,
41+
'@aws-cdk/cli-lib-alpha': alphaInstallationVersion,
42+
'@aws-cdk/aws-lambda-go-alpha': alphaInstallationVersion,
43+
'@aws-cdk/aws-lambda-python-alpha': alphaInstallationVersion,
44+
'constructs': '^10',
45+
});
46+
47+
await block(fixture);
48+
} catch (e: any) {
49+
// We survive certain cases involving gopkg.in
50+
if (errorCausedByGoPkg(e.message)) {
51+
return;
52+
}
53+
success = false;
54+
throw e;
55+
} finally {
56+
if (process.env.INTEG_NO_CLEAN) {
57+
context.log(`Left test directory in '${integTestDir}' ($INTEG_NO_CLEAN)\n`);
58+
} else {
59+
await fixture.dispose(success);
60+
}
61+
}
62+
};
63+
}
64+
65+
/**
66+
* Return whether or not the error is being caused by gopkg.in being down
67+
*
68+
* Our Go build depends on https://gopkg.in/, which has errors pretty often
69+
* (every couple of days). It is run by a single volunteer.
70+
*/
71+
function errorCausedByGoPkg(error: string) {
72+
// The error is different depending on what request fails. Messages recognized:
73+
////////////////////////////////////////////////////////////////////
74+
// go: github.com/aws/[email protected] requires
75+
// gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776: invalid version: git ls-remote -q origin in /go/pkg/mod/cache/vcs/0901dc1ef67fcce1c9b3ae51078740de4a0e2dc673e720584ac302973af82f36: exit status 128:
76+
// remote: Cannot obtain refs from GitHub: cannot talk to GitHub: Get https://github.com/go-yaml/yaml.git/info/refs?service=git-upload-pack: net/http: request canceled (Client.Timeout exceeded while awaiting headers)
77+
// fatal: unable to access 'https://gopkg.in/yaml.v3/': The requested URL returned error: 502
78+
////////////////////////////////////////////////////////////////////
79+
// go: downloading github.com/aws/aws-lambda-go v1.28.0
80+
// go: github.com/aws/[email protected] requires
81+
// gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776: unrecognized import path "gopkg.in/yaml.v3": reading https://gopkg.in/yaml.v3?go-get=1: 502 Bad Gateway
82+
// server response: Cannot obtain refs from GitHub: cannot talk to GitHub: Get https://github.com/go-yaml/yaml.git/info/refs?service=git-upload-pack: net/http: request canceled (Client.Timeout exceeded while awaiting headers)
83+
////////////////////////////////////////////////////////////////////
84+
// go: github.com/aws/[email protected] requires
85+
// gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776: invalid version: git fetch -f origin refs/heads/*:refs/heads/* refs/tags/*:refs/tags/* in /go/pkg/mod/cache/vcs/0901dc1ef67fcce1c9b3ae51078740de4a0e2dc673e720584ac302973af82f36: exit status 128:
86+
// error: RPC failed; HTTP 502 curl 22 The requested URL returned error: 502
87+
// fatal: the remote end hung up unexpectedly
88+
////////////////////////////////////////////////////////////////////
89+
90+
return (error.includes('gopkg\.in.*invalid version.*exit status 128')
91+
|| error.match(/unrecognized import path[^\n]gopkg\.in/));
92+
}
93+
94+
/**
95+
* SAM Integration test fixture for CDK - SAM integration test cases
96+
*/
97+
export function withCliLibFixture(block: (context: CliLibIntegrationTestFixture) => Promise<void>) {
98+
return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withCliLibIntegrationCdkApp(block)));
99+
}
100+
101+
export class CliLibIntegrationTestFixture extends TestFixture {
102+
/**
103+
*
104+
*/
105+
public async cdk(args: string[], options: CdkCliOptions = {}) {
106+
const action = args[0];
107+
const stackName = args[1];
108+
109+
const cliOpts: Record<string, any> = {
110+
stacks: stackName ? [stackName] : undefined,
111+
};
112+
113+
if (action === 'deploy') {
114+
cliOpts.requireApproval = options.neverRequireApproval ? 'never' : 'broadening';
115+
}
116+
117+
return this.shell(['node', '--input-type=module', `<<__EOS__
118+
import { AwsCdkCli } from '@aws-cdk/cli-lib-alpha';
119+
const cli = AwsCdkCli.fromCdkAppDirectory();
120+
121+
await cli.${action}(${JSON.stringify(cliOpts)});
122+
__EOS__`], {
123+
...options,
124+
modEnv: {
125+
AWS_REGION: this.aws.region,
126+
AWS_DEFAULT_REGION: this.aws.region,
127+
STACK_NAME_PREFIX: this.stackNamePrefix,
128+
PACKAGE_LAYOUT_VERSION: this.packages.majorVersion(),
129+
...options.modEnv,
130+
},
131+
});
132+
}
133+
134+
}

‎packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/app.js

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
const cdk = require('aws-cdk-lib/core');
2+
const iam = require('aws-cdk-lib/aws-iam');
3+
const sqs = require('aws-cdk-lib/aws-sqs');
4+
5+
const stackPrefix = process.env.STACK_NAME_PREFIX;
6+
if (!stackPrefix) {
7+
throw new Error(`the STACK_NAME_PREFIX environment variable is required`);
8+
}
9+
10+
class SimpleStack extends cdk.Stack {
11+
constructor(scope, id, props) {
12+
super(scope, id, props);
13+
const queue = new sqs.Queue(this, 'queue', {
14+
visibilityTimeout: cdk.Duration.seconds(300),
15+
});
16+
const role = new iam.Role(this, 'role', {
17+
assumedBy: new iam.AnyPrincipal(),
18+
});
19+
queue.grantConsumeMessages(role);
20+
}
21+
}
22+
23+
const app = new cdk.App();
24+
new SimpleStack(app, `${stackPrefix}-simple-1`);
25+
26+
app.synth();

‎packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/cdk.json

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"app": "node app.js",
3+
"versionReporting": false,
4+
"context": {
5+
"aws-cdk:enableDiffNoFail": "true"
6+
}
7+
}

‎packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib.integtest.ts

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import { integTest, withCliLibFixture } from '../../lib';
2+
3+
jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
4+
5+
integTest('cli-lib synth', withCliLibFixture(async (fixture) => {
6+
await fixture.cdk(['synth', fixture.fullStackName('simple-1')]);
7+
expect(fixture.template('simple-1')).toEqual(expect.objectContaining({
8+
// Checking for a small subset is enough as proof that synth worked
9+
Resources: expect.objectContaining({
10+
queue276F7297: expect.objectContaining({
11+
Type: 'AWS::SQS::Queue',
12+
Properties: {
13+
VisibilityTimeout: 300,
14+
},
15+
Metadata: {
16+
'aws:cdk:path': `${fixture.stackNamePrefix}-simple-1/queue/Resource`,
17+
},
18+
}),
19+
}),
20+
}));
21+
}));
22+
23+
integTest('cli-lib list', withCliLibFixture(async (fixture) => {
24+
const listing = await fixture.cdk(['list'], { captureStderr: false });
25+
expect(listing).toContain(fixture.fullStackName('simple-1'));
26+
}));
27+
28+
integTest('cli-lib deploy', withCliLibFixture(async (fixture) => {
29+
const stackName = fixture.fullStackName('simple-1');
30+
31+
try {
32+
// deploy the stack
33+
await fixture.cdk(['deploy', stackName], {
34+
neverRequireApproval: true,
35+
});
36+
37+
// verify the number of resources in the stack
38+
const expectedStack = await fixture.aws.cloudFormation('describeStackResources', {
39+
StackName: stackName,
40+
});
41+
expect(expectedStack.StackResources?.length).toEqual(3);
42+
} finally {
43+
// delete the stack
44+
await fixture.cdk(['destroy', stackName], {
45+
captureStderr: false,
46+
});
47+
}
48+
}));
49+
50+
integTest('security related changes without a CLI are expected to fail when approval is required', withCliLibFixture(async (fixture) => {
51+
const stdErr = await fixture.cdk(['deploy', fixture.fullStackName('simple-1')], {
52+
onlyStderr: true,
53+
captureStderr: true,
54+
allowErrExit: true,
55+
neverRequireApproval: false,
56+
});
57+
58+
expect(stdErr).toContain('This deployment will make potentially sensitive changes according to your current security approval level');
59+
expect(stdErr).toContain('Deployment failed: Error: \"--require-approval\" is enabled and stack includes security-sensitive updates');
60+
61+
// Ensure stack was not deployed
62+
await expect(fixture.aws.cloudFormation('describeStacks', {
63+
StackName: fixture.fullStackName('simple-1'),
64+
})).rejects.toThrow('does not exist');
65+
}));

‎packages/@aws-cdk/cli-lib-alpha/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
*.js
22
*.js.map
33
*.d.ts
4+
*.gz
45
!lib/init-templates/**/javascript/**/*
56
node_modules
67
dist

‎packages/@aws-cdk/cli-lib-alpha/.npmignore

+1
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,4 @@ tsconfig.json
3535
!build-info.json
3636
!NOTICE
3737
!THIRD_PARTY_LICENSES
38+
!db.json.gz

‎packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES

+3-3
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ The @aws-cdk/cli-lib-alpha package includes the following third-party software/l
207207

208208
----------------
209209

210-
** @jsii/check-node@1.91.0 - https://www.npmjs.com/package/@jsii/check-node/v/1.91.0 | Apache-2.0
210+
** @jsii/check-node@1.92.0 - https://www.npmjs.com/package/@jsii/check-node/v/1.92.0 | Apache-2.0
211211
jsii
212212
Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
213213

@@ -471,7 +471,7 @@ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH RE
471471

472472
----------------
473473

474-
** aws-sdk@2.1492.0 - https://www.npmjs.com/package/aws-sdk/v/2.1492.0 | Apache-2.0
474+
** aws-sdk@2.1498.0 - https://www.npmjs.com/package/aws-sdk/v/2.1498.0 | Apache-2.0
475475
AWS SDK for JavaScript
476476
Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
477477

@@ -668,7 +668,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
668668

669669
----------------
670670

671-
** cdk-from-cfn@0.67.0 - https://www.npmjs.com/package/cdk-from-cfn/v/0.67.0 | MIT OR Apache-2.0
671+
** cdk-from-cfn@0.69.0 - https://www.npmjs.com/package/cdk-from-cfn/v/0.69.0 | MIT OR Apache-2.0
672672

673673
----------------
674674

‎packages/@aws-cdk/cli-lib-alpha/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
"yarn attribute",
6666
"mkdir -p ./lib/api/bootstrap/ && cp ../../aws-cdk/lib/api/bootstrap/bootstrap-template.yaml ./lib/api/bootstrap/",
6767
"cp ../../../node_modules/cdk-from-cfn/index_bg.wasm ./lib/",
68+
"cp ../../../node_modules/@aws-cdk/aws-service-spec/db.json.gz ./",
6869
"yarn bundle",
6970
"node ./lib/main.js >/dev/null 2>/dev/null </dev/null"
7071
],

0 commit comments

Comments
 (0)