Skip to content

Commit d969ddf

Browse files
lpizzinidevlpizzinidev
authored
feat(s3-deployment): added Source.dataYaml helper function (#24579)
Add the `Source.dataYaml` helper function to deploy an object with the specified object converted to YAML format. Closes #24554. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 72b3a95 commit d969ddf

File tree

21 files changed

+421
-87
lines changed

21 files changed

+421
-87
lines changed

Diff for: ‎package.json

+2
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@
119119
"@aws-cdk/core/minimatch/**",
120120
"@aws-cdk/core/table",
121121
"@aws-cdk/core/table/**",
122+
"@aws-cdk/core/yaml",
123+
"@aws-cdk/core/yaml/**",
122124
"@aws-cdk/cx-api/semver",
123125
"@aws-cdk/cx-api/semver/**",
124126
"@aws-cdk/pipelines/aws-sdk",

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/README.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ The following source types are supported for bucket deployments:
8686
(supports [deploy-time values](#data-with-deploy-time-values))
8787
- JSON data: `s3deploy.Source.jsonData('object-key.json', { json: 'object' })`
8888
(supports [deploy-time values](#data-with-deploy-time-values))
89+
- YAML data: `s3deploy.Source.yamlData('object-key.yaml', { yaml: 'object' })`
90+
(supports [deploy-time values](#data-with-deploy-time-values))
8991

9092
To create a source from a single file, you can pass `AssetOptions` to exclude
9193
all but a single file:
@@ -313,7 +315,7 @@ new s3deploy.BucketDeployment(this, 'DeployMeWithEfsStorage', {
313315

314316
## Data with deploy-time values
315317

316-
The content passed to `Source.data()` or `Source.jsonData()` can include
318+
The content passed to `Source.data()`, `Source.jsonData()`, or `Source.yamlData()` can include
317319
references that will get resolved only during deployment.
318320

319321
For example:

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/lib/source.ts

+20-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ export interface ISource {
5858
* Source.asset('/local/path/to/directory')
5959
* Source.asset('/local/path/to/a/file.zip')
6060
* Source.data('hello/world/file.txt', 'Hello, world!')
61-
* Source.data('config.json', { baz: topic.topicArn })
61+
* Source.dataJson('config.json', { baz: topic.topicArn })
62+
* Source.dataYaml('config.yaml', { baz: topic.topicArn })
6263
*
6364
*/
6465
export class Source {
@@ -125,6 +126,7 @@ export class Source {
125126
* will get resolved only during deployment.
126127
*
127128
* To store a JSON object use `Source.jsonData()`.
129+
* To store YAML content use `Source.yamlData()`.
128130
*
129131
* @param objectKey The destination S3 object key (relative to the root of the
130132
* S3 deployment).
@@ -165,5 +167,22 @@ export class Source {
165167
};
166168
}
167169

170+
/**
171+
* Deploys an object with the specified JSON object formatted as YAML into the bucket.
172+
* The object can include deploy-time values (such as `snsTopic.topicArn`) that
173+
* will get resolved only during deployment.
174+
*
175+
* @param objectKey The destination S3 object key (relative to the root of the
176+
* S3 deployment).
177+
* @param obj A JSON object.
178+
*/
179+
public static yamlData(objectKey: string, obj: any): ISource {
180+
return {
181+
bind: (scope: Construct, context?: DeploymentSourceContext) => {
182+
return Source.data(objectKey, Stack.of(scope).toYamlString(obj)).bind(scope, context);
183+
},
184+
};
185+
}
186+
168187
private constructor() { }
169188
}

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts

+26
Original file line numberDiff line numberDiff line change
@@ -1387,6 +1387,32 @@ test('Source.jsonData() can be used to create a file with a JSON object', () =>
13871387
});
13881388
});
13891389

1390+
test('Source.yamlData() can be used to create a file with YAML content', () => {
1391+
const app = new cdk.App();
1392+
const stack = new cdk.Stack(app, 'Test');
1393+
const bucket = new s3.Bucket(stack, 'Bucket');
1394+
1395+
const config = {
1396+
foo: 'bar',
1397+
sub: {
1398+
hello: bucket.bucketArn,
1399+
},
1400+
};
1401+
1402+
new s3deploy.BucketDeployment(stack, 'DeployWithVpc3', {
1403+
sources: [s3deploy.Source.yamlData('app-config.yaml', config)],
1404+
destinationBucket: bucket,
1405+
});
1406+
1407+
const result = app.synth();
1408+
const output = readDataFile(result, 'app-config.yaml');
1409+
expect(output.trim()).toEqual([
1410+
'foo: bar',
1411+
'sub:',
1412+
' hello: <<marker:0xbaba:0>>',
1413+
].join('\n'));
1414+
});
1415+
13901416
test('can add sources with addSource', () => {
13911417
const app = new cdk.App();
13921418
const stack = new cdk.Stack(app, 'Test');

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/TestBucketDeploymentContent.assets.json

+25-12
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
11
{
2-
"version": "20.0.0",
2+
"version": "31.0.0",
33
"files": {
4-
"39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080": {
4+
"68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a": {
55
"source": {
6-
"path": "asset.39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080.zip",
6+
"path": "asset.68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a.zip",
77
"packaging": "file"
88
},
99
"destinations": {
1010
"current_account-current_region": {
1111
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
12-
"objectKey": "39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080.zip",
12+
"objectKey": "68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a.zip",
1313
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
1414
}
1515
}
1616
},
17-
"f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da": {
17+
"2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64": {
1818
"source": {
19-
"path": "asset.f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da",
19+
"path": "asset.2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64",
2020
"packaging": "zip"
2121
},
2222
"destinations": {
2323
"current_account-current_region": {
2424
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
25-
"objectKey": "f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da.zip",
25+
"objectKey": "2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64.zip",
2626
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
2727
}
2828
}
@@ -53,28 +53,41 @@
5353
}
5454
}
5555
},
56-
"0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a": {
56+
"27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f": {
5757
"source": {
58-
"path": "asset.0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a",
58+
"path": "asset.27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f",
5959
"packaging": "zip"
6060
},
6161
"destinations": {
6262
"current_account-current_region": {
6363
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
64-
"objectKey": "0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a.zip",
64+
"objectKey": "27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f.zip",
6565
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
6666
}
6767
}
6868
},
69-
"55549864d8c27c2125682e2e817e0a7fede6b77576ccd6e21178645f08472948": {
69+
"939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f": {
70+
"source": {
71+
"path": "asset.939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f",
72+
"packaging": "zip"
73+
},
74+
"destinations": {
75+
"current_account-current_region": {
76+
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
77+
"objectKey": "939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f.zip",
78+
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
79+
}
80+
}
81+
},
82+
"2961e8222a48394849f4466d2789ae256aa88adc4ccbf79feb35306b850c08dc": {
7083
"source": {
7184
"path": "TestBucketDeploymentContent.template.json",
7285
"packaging": "file"
7386
},
7487
"destinations": {
7588
"current_account-current_region": {
7689
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
77-
"objectKey": "55549864d8c27c2125682e2e817e0a7fede6b77576ccd6e21178645f08472948.json",
90+
"objectKey": "2961e8222a48394849f4466d2789ae256aa88adc4ccbf79feb35306b850c08dc.json",
7891
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
7992
}
8093
}

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/TestBucketDeploymentContent.template.json

+15-3
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
"S3Bucket": {
2121
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
2222
},
23-
"S3Key": "39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080.zip"
23+
"S3Key": "68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a.zip"
2424
},
2525
"Description": "/opt/awscli/aws"
2626
}
@@ -41,14 +41,18 @@
4141
{
4242
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
4343
},
44+
{
45+
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
46+
},
4447
{
4548
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
4649
}
4750
],
4851
"SourceObjectKeys": [
4952
"d09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18d.zip",
5053
"0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936b.zip",
51-
"0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a.zip"
54+
"27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f.zip",
55+
"939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f.zip"
5256
],
5357
"SourceMarkers": [
5458
{},
@@ -57,6 +61,14 @@
5761
"Ref": "Bucket83908E77"
5862
}
5963
},
64+
{
65+
"<<marker:0xbaba:0>>": {
66+
"Fn::GetAtt": [
67+
"Bucket83908E77",
68+
"WebsiteURL"
69+
]
70+
}
71+
},
6072
{
6173
"<<marker:0xbaba:0>>": {
6274
"Fn::GetAtt": [
@@ -208,7 +220,7 @@
208220
"S3Bucket": {
209221
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
210222
},
211-
"S3Key": "f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da.zip"
223+
"S3Key": "2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64.zip"
212224
},
213225
"Role": {
214226
"Fn::GetAtt": [

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/asset.0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a/my/config.json renamed to ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/asset.27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f/my-json/config.json

File renamed without changes.

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/asset.f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da/index.py renamed to ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/asset.2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64/index.py

+17-12
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ def cfn_error(message=None):
4949
source_markers = props.get('SourceMarkers', None)
5050
dest_bucket_name = props['DestinationBucketName']
5151
dest_bucket_prefix = props.get('DestinationBucketKeyPrefix', '')
52+
extract = props.get('Extract', 'true') == 'true'
5253
retain_on_delete = props.get('RetainOnDelete', "true") == "true"
5354
distribution_id = props.get('DistributionId', '')
5455
user_metadata = props.get('UserMetadata', {})
@@ -113,14 +114,15 @@ def cfn_error(message=None):
113114
aws_command("s3", "rm", old_s3_dest, "--recursive")
114115

115116
if request_type == "Update" or request_type == "Create":
116-
s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers)
117+
s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers, extract)
117118

118119
if distribution_id:
119120
cloudfront_invalidate(distribution_id, distribution_paths)
120121

121122
cfn_send(event, context, CFN_SUCCESS, physicalResourceId=physical_id, responseData={
122123
# Passing through the ARN sequences dependencees on the deployment
123-
'DestinationBucketArn': props.get('DestinationBucketArn')
124+
'DestinationBucketArn': props.get('DestinationBucketArn'),
125+
'SourceObjectKeys': props.get('SourceObjectKeys'),
124126
})
125127
except KeyError as e:
126128
cfn_error("invalid request. Missing key %s" % str(e))
@@ -130,7 +132,7 @@ def cfn_error(message=None):
130132

131133
#---------------------------------------------------------------------------------------------------
132134
# populate all files from s3_source_zips to a destination bucket
133-
def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers):
135+
def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers, extract):
134136
# list lengths are equal
135137
if len(s3_source_zips) != len(source_markers):
136138
raise Exception("'source_markers' and 's3_source_zips' must be the same length")
@@ -154,12 +156,16 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex
154156
s3_source_zip = s3_source_zips[i]
155157
markers = source_markers[i]
156158

157-
archive=os.path.join(workdir, str(uuid4()))
158-
logger.info("archive: %s" % archive)
159-
aws_command("s3", "cp", s3_source_zip, archive)
160-
logger.info("| extracting archive to: %s\n" % contents_dir)
161-
logger.info("| markers: %s" % markers)
162-
extract_and_replace_markers(archive, contents_dir, markers)
159+
if extract:
160+
archive=os.path.join(workdir, str(uuid4()))
161+
logger.info("archive: %s" % archive)
162+
aws_command("s3", "cp", s3_source_zip, archive)
163+
logger.info("| extracting archive to: %s\n" % contents_dir)
164+
logger.info("| markers: %s" % markers)
165+
extract_and_replace_markers(archive, contents_dir, markers)
166+
else:
167+
logger.info("| copying archive to: %s\n" % contents_dir)
168+
aws_command("s3", "cp", s3_source_zip, contents_dir)
163169

164170
# sync from "contents" to destination
165171

@@ -230,7 +236,6 @@ def aws_command(*args):
230236
def cfn_send(event, context, responseStatus, responseData={}, physicalResourceId=None, noEcho=False, reason=None):
231237

232238
responseUrl = event['ResponseURL']
233-
logger.info(responseUrl)
234239

235240
responseBody = {}
236241
responseBody['Status'] = responseStatus
@@ -285,7 +290,7 @@ def extract_and_replace_markers(archive, contents_dir, markers):
285290
for file in zip.namelist():
286291
file_path = os.path.join(contents_dir, file)
287292
if os.path.isdir(file_path): continue
288-
replace_markers(file_path, markers)
293+
replace_markers(file_path, markers)
289294

290295
def replace_markers(filename, markers):
291296
# convert the dict of string markers to binary markers
@@ -300,4 +305,4 @@ def replace_markers(filename, markers):
300305

301306
# # delete the original file and rename the new one to the original
302307
os.remove(filename)
303-
os.rename(outfile, filename)
308+
os.rename(outfile, filename)

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/asset.39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080.zip renamed to ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/asset.68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a.zip

13.3 MB
Binary file not shown.

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/asset.939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f/my-yaml/config.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
website_url: <<marker:0xbaba:0>>

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/cdk.out

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"version":"21.0.0"}
1+
{"version":"31.0.0"}

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/integ.json

+5-7
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
{
2-
"version": "21.0.0",
2+
"version": "31.0.0",
33
"testCases": {
4-
"integ.bucket-deployment-data": {
4+
"integ-test-bucket-deployment-data/DefaultTest": {
55
"stacks": [
66
"TestBucketDeploymentContent"
77
],
8-
"diffAssets": false,
9-
"stackUpdateWorkflow": true
8+
"assertionStack": "integ-test-bucket-deployment-data/DefaultTest/DeployAssert",
9+
"assertionStackName": "integtestbucketdeploymentdataDefaultTestDeployAssert6FF3075D"
1010
}
11-
},
12-
"synthContext": {},
13-
"enableLookups": false
11+
}
1412
}

Diff for: ‎packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/integtestbucketdeploymentdataDefaultTestDeployAssert6FF3075D.assets.json

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"version": "31.0.0",
3+
"files": {
4+
"21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
5+
"source": {
6+
"path": "integtestbucketdeploymentdataDefaultTestDeployAssert6FF3075D.template.json",
7+
"packaging": "file"
8+
},
9+
"destinations": {
10+
"current_account-current_region": {
11+
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
12+
"objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
13+
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
14+
}
15+
}
16+
}
17+
},
18+
"dockerImages": {}
19+
}

0 commit comments

Comments
 (0)