Skip to content

Commit 4ff3565

Browse files
moomindanimoomindani
authored
fix(glue): s3 path specified in --spark-event-logs-path needs to end with slash (#29357)
### Issue # (if applicable) Closes #29356. ### Reason for this change Currently S3 path specified in `--spark-event-logs-path` does not end with slash in case only bucket is provided but prefix is not provided. This parameter causes errors when viewing the event log through Spark UI / Spark history server. ### Description of changes Add trailing slash when it does not end with slash. ### Description of how you validated changes Completed unit test and integ test. ### Checklist - [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent f0d1d67 commit 4ff3565

File tree

7 files changed

+420
-11
lines changed

7 files changed

+420
-11
lines changed

‎packages/@aws-cdk/aws-glue-alpha/lib/job.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -826,7 +826,7 @@ export class Job extends JobBase {
826826
bucket.grantReadWrite(role, this.cleanPrefixForGrant(props.prefix));
827827
const args = {
828828
'--enable-spark-ui': 'true',
829-
'--spark-event-logs-path': bucket.s3UrlForObject(props.prefix),
829+
'--spark-event-logs-path': bucket.s3UrlForObject(props.prefix).replace(/\/?$/, '/'), // path will always end with a slash
830830
};
831831

832832
return {

‎packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json

Lines changed: 171 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@
162162
"s3://",
163163
{
164164
"Ref": "EtlJob20SparkUIBucketFD07FBD8"
165-
}
165+
},
166+
"/"
166167
]
167168
]
168169
},
@@ -267,6 +268,43 @@
267268
"Properties": {
268269
"PolicyDocument": {
269270
"Statement": [
271+
{
272+
"Action": [
273+
"s3:Abort*",
274+
"s3:DeleteObject*",
275+
"s3:GetBucket*",
276+
"s3:GetObject*",
277+
"s3:List*",
278+
"s3:PutObject",
279+
"s3:PutObjectLegalHold",
280+
"s3:PutObjectRetention",
281+
"s3:PutObjectTagging",
282+
"s3:PutObjectVersionTagging"
283+
],
284+
"Effect": "Allow",
285+
"Resource": [
286+
{
287+
"Fn::GetAtt": [
288+
"StreamingJob20SparkUIBucket92EF3706",
289+
"Arn"
290+
]
291+
},
292+
{
293+
"Fn::Join": [
294+
"",
295+
[
296+
{
297+
"Fn::GetAtt": [
298+
"StreamingJob20SparkUIBucket92EF3706",
299+
"Arn"
300+
]
301+
},
302+
"/*"
303+
]
304+
]
305+
}
306+
]
307+
},
270308
{
271309
"Action": [
272310
"s3:GetBucket*",
@@ -319,6 +357,11 @@
319357
]
320358
}
321359
},
360+
"StreamingJob20SparkUIBucket92EF3706": {
361+
"Type": "AWS::S3::Bucket",
362+
"UpdateReplacePolicy": "Retain",
363+
"DeletionPolicy": "Retain"
364+
},
322365
"StreamingJob20355B58C7": {
323366
"Type": "AWS::Glue::Job",
324367
"Properties": {
@@ -340,6 +383,19 @@
340383
},
341384
"DefaultArguments": {
342385
"--job-language": "python",
386+
"--enable-spark-ui": "true",
387+
"--spark-event-logs-path": {
388+
"Fn::Join": [
389+
"",
390+
[
391+
"s3://",
392+
{
393+
"Ref": "StreamingJob20SparkUIBucket92EF3706"
394+
},
395+
"/"
396+
]
397+
]
398+
},
343399
"arg1": "value1",
344400
"arg2": "value2"
345401
},
@@ -520,7 +576,8 @@
520576
"s3://",
521577
{
522578
"Ref": "EtlJob30SparkUIBucket9D789346"
523-
}
579+
},
580+
"/"
524581
]
525582
]
526583
},
@@ -625,6 +682,43 @@
625682
"Properties": {
626683
"PolicyDocument": {
627684
"Statement": [
685+
{
686+
"Action": [
687+
"s3:Abort*",
688+
"s3:DeleteObject*",
689+
"s3:GetBucket*",
690+
"s3:GetObject*",
691+
"s3:List*",
692+
"s3:PutObject",
693+
"s3:PutObjectLegalHold",
694+
"s3:PutObjectRetention",
695+
"s3:PutObjectTagging",
696+
"s3:PutObjectVersionTagging"
697+
],
698+
"Effect": "Allow",
699+
"Resource": [
700+
{
701+
"Fn::GetAtt": [
702+
"StreamingJob30SparkUIBucketEFBF52D3",
703+
"Arn"
704+
]
705+
},
706+
{
707+
"Fn::Join": [
708+
"",
709+
[
710+
{
711+
"Fn::GetAtt": [
712+
"StreamingJob30SparkUIBucketEFBF52D3",
713+
"Arn"
714+
]
715+
},
716+
"/*"
717+
]
718+
]
719+
}
720+
]
721+
},
628722
{
629723
"Action": [
630724
"s3:GetBucket*",
@@ -677,6 +771,11 @@
677771
]
678772
}
679773
},
774+
"StreamingJob30SparkUIBucketEFBF52D3": {
775+
"Type": "AWS::S3::Bucket",
776+
"UpdateReplacePolicy": "Retain",
777+
"DeletionPolicy": "Retain"
778+
},
680779
"StreamingJob30E005FBEB": {
681780
"Type": "AWS::Glue::Job",
682781
"Properties": {
@@ -698,6 +797,19 @@
698797
},
699798
"DefaultArguments": {
700799
"--job-language": "python",
800+
"--enable-spark-ui": "true",
801+
"--spark-event-logs-path": {
802+
"Fn::Join": [
803+
"",
804+
[
805+
"s3://",
806+
{
807+
"Ref": "StreamingJob30SparkUIBucketEFBF52D3"
808+
},
809+
"/"
810+
]
811+
]
812+
},
701813
"arg1": "value1",
702814
"arg2": "value2"
703815
},
@@ -878,7 +990,8 @@
878990
"s3://",
879991
{
880992
"Ref": "EtlJob40SparkUIBucket02F50B0D"
881-
}
993+
},
994+
"/"
882995
]
883996
]
884997
},
@@ -983,6 +1096,43 @@
9831096
"Properties": {
9841097
"PolicyDocument": {
9851098
"Statement": [
1099+
{
1100+
"Action": [
1101+
"s3:Abort*",
1102+
"s3:DeleteObject*",
1103+
"s3:GetBucket*",
1104+
"s3:GetObject*",
1105+
"s3:List*",
1106+
"s3:PutObject",
1107+
"s3:PutObjectLegalHold",
1108+
"s3:PutObjectRetention",
1109+
"s3:PutObjectTagging",
1110+
"s3:PutObjectVersionTagging"
1111+
],
1112+
"Effect": "Allow",
1113+
"Resource": [
1114+
{
1115+
"Fn::GetAtt": [
1116+
"StreamingJob40SparkUIBucketA97E24C6",
1117+
"Arn"
1118+
]
1119+
},
1120+
{
1121+
"Fn::Join": [
1122+
"",
1123+
[
1124+
{
1125+
"Fn::GetAtt": [
1126+
"StreamingJob40SparkUIBucketA97E24C6",
1127+
"Arn"
1128+
]
1129+
},
1130+
"/*"
1131+
]
1132+
]
1133+
}
1134+
]
1135+
},
9861136
{
9871137
"Action": [
9881138
"s3:GetBucket*",
@@ -1035,6 +1185,11 @@
10351185
]
10361186
}
10371187
},
1188+
"StreamingJob40SparkUIBucketA97E24C6": {
1189+
"Type": "AWS::S3::Bucket",
1190+
"UpdateReplacePolicy": "Retain",
1191+
"DeletionPolicy": "Retain"
1192+
},
10381193
"StreamingJob40E284A782": {
10391194
"Type": "AWS::Glue::Job",
10401195
"Properties": {
@@ -1056,6 +1211,19 @@
10561211
},
10571212
"DefaultArguments": {
10581213
"--job-language": "python",
1214+
"--enable-spark-ui": "true",
1215+
"--spark-event-logs-path": {
1216+
"Fn::Join": [
1217+
"",
1218+
[
1219+
"s3://",
1220+
{
1221+
"Ref": "StreamingJob40SparkUIBucketA97E24C6"
1222+
},
1223+
"/"
1224+
]
1225+
]
1226+
},
10591227
"arg1": "value1",
10601228
"arg2": "value2"
10611229
},

‎packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json

Lines changed: 19 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)