Skip to content

Commit 2952408

Browse files
authored
feat(s3): date-based partitioning for log objects (#28790)
S3 server access logging newly supports date-based partitioning. This feature allows key formats of log objects to be partitioned by date, which is useful when querying from Athena. In CloudFormation, `TargetObjectKeyFormat` was added. This object has the `PartitionedPrefix` and the `SimplePrefix` property, only one of which is allowed. The `SimplePrefix` is the key format that originally existed. The `PartitionedPrefix` is the new key format and allows the user to select either the event time or delivery time as the date source. ```yaml Resources: S3BucketServerAccessLogEnabledTest: Type: AWS::S3::Bucket Properties: BucketName: access-log-test LoggingConfiguration: DestinationBucketName: myDestinationBucket LogFilePrefix: 'hello' TargetObjectKeyFormat: # You can deploy in either of the following formats SimplePrefix: {} # 1 PartitionedPrefix: {} # 2 PartitionedPrefix: # 3 PartitionDateSource: EventTime # | DeliveryTime ``` whats-new https://aws.amazon.com/about-aws/whats-new/2023/11/amazon-s3-server-access-logging-date-partitioning CloudFormation User Guides https://docs.aws.amazon.com/ja_jp/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket-loggingconfiguration.html#cfn-s3-bucket-loggingconfiguration-targetobjectkeyformat S3 User Guides https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html#server-access-logging-overview Closes #28141 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 7a30f5d commit 2952408

File tree

11 files changed

+746
-26
lines changed

11 files changed

+746
-26
lines changed

packages/@aws-cdk-testing/framework-integ/test/aws-s3/test/integ.bucket.server-access-logs.js.snapshot/aws-cdk-s3-access-logs.assets.json

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk-testing/framework-integ/test/aws-s3/test/integ.bucket.server-access-logs.js.snapshot/aws-cdk-s3-access-logs.template.json

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,150 @@
4848
]
4949
]
5050
}
51+
},
52+
{
53+
"Action": "s3:PutObject",
54+
"Condition": {
55+
"ArnLike": {
56+
"aws:SourceArn": {
57+
"Fn::GetAtt": [
58+
"MyBucket26E0C3623",
59+
"Arn"
60+
]
61+
}
62+
},
63+
"StringEquals": {
64+
"aws:SourceAccount": {
65+
"Ref": "AWS::AccountId"
66+
}
67+
}
68+
},
69+
"Effect": "Allow",
70+
"Principal": {
71+
"Service": "logging.s3.amazonaws.com"
72+
},
73+
"Resource": {
74+
"Fn::Join": [
75+
"",
76+
[
77+
{
78+
"Fn::GetAtt": [
79+
"MyAccessLogsBucketF7FE6635",
80+
"Arn"
81+
]
82+
},
83+
"/example2*"
84+
]
85+
]
86+
}
87+
},
88+
{
89+
"Action": "s3:PutObject",
90+
"Condition": {
91+
"ArnLike": {
92+
"aws:SourceArn": {
93+
"Fn::GetAtt": [
94+
"MyBucket3CC4F8735",
95+
"Arn"
96+
]
97+
}
98+
},
99+
"StringEquals": {
100+
"aws:SourceAccount": {
101+
"Ref": "AWS::AccountId"
102+
}
103+
}
104+
},
105+
"Effect": "Allow",
106+
"Principal": {
107+
"Service": "logging.s3.amazonaws.com"
108+
},
109+
"Resource": {
110+
"Fn::Join": [
111+
"",
112+
[
113+
{
114+
"Fn::GetAtt": [
115+
"MyAccessLogsBucketF7FE6635",
116+
"Arn"
117+
]
118+
},
119+
"/example3*"
120+
]
121+
]
122+
}
123+
},
124+
{
125+
"Action": "s3:PutObject",
126+
"Condition": {
127+
"ArnLike": {
128+
"aws:SourceArn": {
129+
"Fn::GetAtt": [
130+
"MyBucket43E0A113B",
131+
"Arn"
132+
]
133+
}
134+
},
135+
"StringEquals": {
136+
"aws:SourceAccount": {
137+
"Ref": "AWS::AccountId"
138+
}
139+
}
140+
},
141+
"Effect": "Allow",
142+
"Principal": {
143+
"Service": "logging.s3.amazonaws.com"
144+
},
145+
"Resource": {
146+
"Fn::Join": [
147+
"",
148+
[
149+
{
150+
"Fn::GetAtt": [
151+
"MyAccessLogsBucketF7FE6635",
152+
"Arn"
153+
]
154+
},
155+
"/example4*"
156+
]
157+
]
158+
}
159+
},
160+
{
161+
"Action": "s3:PutObject",
162+
"Condition": {
163+
"ArnLike": {
164+
"aws:SourceArn": {
165+
"Fn::GetAtt": [
166+
"MyBucket53983D51A",
167+
"Arn"
168+
]
169+
}
170+
},
171+
"StringEquals": {
172+
"aws:SourceAccount": {
173+
"Ref": "AWS::AccountId"
174+
}
175+
}
176+
},
177+
"Effect": "Allow",
178+
"Principal": {
179+
"Service": "logging.s3.amazonaws.com"
180+
},
181+
"Resource": {
182+
"Fn::Join": [
183+
"",
184+
[
185+
{
186+
"Fn::GetAtt": [
187+
"MyAccessLogsBucketF7FE6635",
188+
"Arn"
189+
]
190+
},
191+
"/example5*"
192+
]
193+
]
194+
}
51195
}
52196
],
53197
"Version": "2012-10-17"
@@ -66,6 +210,74 @@
66210
},
67211
"UpdateReplacePolicy": "Delete",
68212
"DeletionPolicy": "Delete"
213+
},
214+
"MyBucket26E0C3623": {
215+
"Type": "AWS::S3::Bucket",
216+
"Properties": {
217+
"LoggingConfiguration": {
218+
"DestinationBucketName": {
219+
"Ref": "MyAccessLogsBucketF7FE6635"
220+
},
221+
"LogFilePrefix": "example2",
222+
"TargetObjectKeyFormat": {
223+
"SimplePrefix": {}
224+
}
225+
}
226+
},
227+
"UpdateReplacePolicy": "Delete",
228+
"DeletionPolicy": "Delete"
229+
},
230+
"MyBucket3CC4F8735": {
231+
"Type": "AWS::S3::Bucket",
232+
"Properties": {
233+
"LoggingConfiguration": {
234+
"DestinationBucketName": {
235+
"Ref": "MyAccessLogsBucketF7FE6635"
236+
},
237+
"LogFilePrefix": "example3",
238+
"TargetObjectKeyFormat": {
239+
"PartitionedPrefix": {
240+
"PartitionDateSource": "EventTime"
241+
}
242+
}
243+
}
244+
},
245+
"UpdateReplacePolicy": "Delete",
246+
"DeletionPolicy": "Delete"
247+
},
248+
"MyBucket43E0A113B": {
249+
"Type": "AWS::S3::Bucket",
250+
"Properties": {
251+
"LoggingConfiguration": {
252+
"DestinationBucketName": {
253+
"Ref": "MyAccessLogsBucketF7FE6635"
254+
},
255+
"LogFilePrefix": "example4",
256+
"TargetObjectKeyFormat": {
257+
"PartitionedPrefix": {
258+
"PartitionDateSource": "DeliveryTime"
259+
}
260+
}
261+
}
262+
},
263+
"UpdateReplacePolicy": "Delete",
264+
"DeletionPolicy": "Delete"
265+
},
266+
"MyBucket53983D51A": {
267+
"Type": "AWS::S3::Bucket",
268+
"Properties": {
269+
"LoggingConfiguration": {
270+
"DestinationBucketName": {
271+
"Ref": "MyAccessLogsBucketF7FE6635"
272+
},
273+
"LogFilePrefix": "example5",
274+
"TargetObjectKeyFormat": {
275+
"PartitionedPrefix": {}
276+
}
277+
}
278+
},
279+
"UpdateReplacePolicy": "Delete",
280+
"DeletionPolicy": "Delete"
69281
}
70282
},
71283
"Parameters": {

packages/@aws-cdk-testing/framework-integ/test/aws-s3/test/integ.bucket.server-access-logs.js.snapshot/cdk.out

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk-testing/framework-integ/test/aws-s3/test/integ.bucket.server-access-logs.js.snapshot/cdkintegs3accesslogsDefaultTestDeployAssert37A16466.assets.json

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk-testing/framework-integ/test/aws-s3/test/integ.bucket.server-access-logs.js.snapshot/integ.json

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/@aws-cdk-testing/framework-integ/test/aws-s3/test/integ.bucket.server-access-logs.js.snapshot/manifest.json

Lines changed: 28 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)