Skip to content

Commit 9774d4c

Browse files
mevemeve
authored
feat(glue): Added value to PythonVersion enum (#21670)
PR to fix #21568. Extended the PythonVersion enum to include 3.9, as it already seems to be supported everywhere (CloudFormation, SDK). #21568 (comment) ---- ### All Submissions: * [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) ### Adding new Unconventional Dependencies: * [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies) ### New Features * [x] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)? * [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)? *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent f9ca639 commit 9774d4c

File tree

9 files changed

+445
-15
lines changed

9 files changed

+445
-15
lines changed

‎packages/@aws-cdk/aws-glue/README.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,11 @@ new glue.Job(this, 'PythonSparkStreamingJob', {
7272
### Python Shell Jobs
7373

7474
A Python shell job runs Python scripts as a shell and supports a Python version that depends on the AWS Glue version you are using.
75-
This can be used to schedule and run tasks that don't require an Apache Spark environment.
75+
This can be used to schedule and run tasks that don't require an Apache Spark environment. Currently, three flavors are supported:
76+
77+
* PythonVersion.TWO (2.7; EOL)
78+
* PythonVersion.THREE (3.6)
79+
* PythonVersion.THREE_NINE (3.9)
7680

7781
```ts
7882
declare const bucket: s3.Bucket;

‎packages/@aws-cdk/aws-glue/lib/job-executable.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ export enum PythonVersion {
7575
* Python 3 (the exact version depends on GlueVersion and JobCommand used)
7676
*/
7777
THREE = '3',
78+
79+
/**
80+
* Python 3.9 (the exact version depends on GlueVersion and JobCommand used)
81+
*/
82+
THREE_NINE = '3.9',
7883
}
7984

8085
/**
@@ -300,6 +305,9 @@ export class JobExecutable {
300305
if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) {
301306
throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON');
302307
}
308+
if (config.pythonVersion === PythonVersion.THREE_NINE && config.type !== JobType.PYTHON_SHELL) {
309+
throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell');
310+
}
303311
this.config = config;
304312
}
305313

‎packages/@aws-cdk/aws-glue/test/integ.job.ts

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,20 @@ new glue.Job(stack, 'ShellJob', {
8686
},
8787
});
8888

89+
new glue.Job(stack, 'ShellJob39', {
90+
jobName: 'ShellJob39',
91+
executable: glue.JobExecutable.pythonShell({
92+
glueVersion: glue.GlueVersion.V1_0,
93+
pythonVersion: glue.PythonVersion.THREE_NINE,
94+
script,
95+
}),
96+
defaultArguments: {
97+
arg1: 'value1',
98+
arg2: 'value2',
99+
},
100+
tags: {
101+
key: 'value',
102+
},
103+
});
104+
89105
app.synth();

‎packages/@aws-cdk/aws-glue/test/job-executable.test.ts

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ describe('GlueVersion', () => {
1414
test('of(customVersion) should set the name correctly', () => expect(glue.GlueVersion.of('CustomVersion').name).toEqual('CustomVersion'));
1515
});
1616

17+
describe('PythonVersion', () => {
18+
test('.TWO should set the name correctly', () => expect(glue.PythonVersion.TWO).toEqual('2'));
19+
20+
test('.THREE should set the name correctly', () => expect(glue.PythonVersion.THREE).toEqual('3'));
21+
22+
test('.THREE_NINE should set the name correctly', () => expect(glue.PythonVersion.THREE_NINE).toEqual('3.9'));
23+
});
24+
1725
describe('JobType', () => {
1826
test('.ETL should set the name correctly', () => expect(glue.JobType.ETL.name).toEqual('glueetl'));
1927

@@ -102,5 +110,25 @@ describe('JobExecutable', () => {
102110
})).toThrow(`Specified GlueVersion ${glueVersion.name} does not support PythonVersion 2`);
103111
});
104112
});
113+
114+
test('with PythonVersion set to PythonVersion.THREE_NINE and JobType not pythonshell should throw', () => {
115+
expect(() => glue.JobExecutable.of({
116+
type: glue.JobType.ETL,
117+
language: glue.JobLanguage.PYTHON,
118+
pythonVersion: glue.PythonVersion.THREE_NINE,
119+
script,
120+
glueVersion: glue.GlueVersion.V1_0,
121+
})).toThrow('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell');
122+
});
123+
124+
test('with PythonVersion PythonVersion.THREE_NINE and JobType pythonshell should succeed', () => {
125+
expect(glue.JobExecutable.of({
126+
type: glue.JobType.PYTHON_SHELL,
127+
glueVersion: glue.GlueVersion.V1_0,
128+
language: glue.JobLanguage.PYTHON,
129+
pythonVersion: glue.PythonVersion.THREE_NINE,
130+
script,
131+
})).toBeDefined();
132+
});
105133
});
106134
});

‎packages/@aws-cdk/aws-glue/test/job.integ.snapshot/aws-glue-job.assets.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "17.0.0",
2+
"version": "20.0.0",
33
"files": {
44
"432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": {
55
"source": {
@@ -14,15 +14,15 @@
1414
}
1515
}
1616
},
17-
"9621659bccc489f1130729575d70f1c13a90dc7584eac90773642d7ad0cc1914": {
17+
"a44de72f7ea39efc17f8c7abc7e0c595ae4651b3e239851c49461fecf09bb151": {
1818
"source": {
1919
"path": "aws-glue-job.template.json",
2020
"packaging": "file"
2121
},
2222
"destinations": {
2323
"current_account-current_region": {
2424
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
25-
"objectKey": "9621659bccc489f1130729575d70f1c13a90dc7584eac90773642d7ad0cc1914.json",
25+
"objectKey": "a44de72f7ea39efc17f8c7abc7e0c595ae4651b3e239851c49461fecf09bb151.json",
2626
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
2727
}
2828
}

‎packages/@aws-cdk/aws-glue/test/job.integ.snapshot/aws-glue-job.template.json

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,157 @@
556556
"key": "value"
557557
}
558558
}
559+
},
560+
"ShellJob39ServiceRole2F6F3768": {
561+
"Type": "AWS::IAM::Role",
562+
"Properties": {
563+
"AssumeRolePolicyDocument": {
564+
"Statement": [
565+
{
566+
"Action": "sts:AssumeRole",
567+
"Effect": "Allow",
568+
"Principal": {
569+
"Service": "glue.amazonaws.com"
570+
}
571+
}
572+
],
573+
"Version": "2012-10-17"
574+
},
575+
"ManagedPolicyArns": [
576+
{
577+
"Fn::Join": [
578+
"",
579+
[
580+
"arn:",
581+
{
582+
"Ref": "AWS::Partition"
583+
},
584+
":iam::aws:policy/service-role/AWSGlueServiceRole"
585+
]
586+
]
587+
}
588+
]
589+
}
590+
},
591+
"ShellJob39ServiceRoleDefaultPolicy38A33919": {
592+
"Type": "AWS::IAM::Policy",
593+
"Properties": {
594+
"PolicyDocument": {
595+
"Statement": [
596+
{
597+
"Action": [
598+
"s3:GetBucket*",
599+
"s3:GetObject*",
600+
"s3:List*"
601+
],
602+
"Effect": "Allow",
603+
"Resource": [
604+
{
605+
"Fn::Join": [
606+
"",
607+
[
608+
"arn:",
609+
{
610+
"Ref": "AWS::Partition"
611+
},
612+
":s3:::",
613+
{
614+
"Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3Bucket4E517469"
615+
},
616+
"/*"
617+
]
618+
]
619+
},
620+
{
621+
"Fn::Join": [
622+
"",
623+
[
624+
"arn:",
625+
{
626+
"Ref": "AWS::Partition"
627+
},
628+
":s3:::",
629+
{
630+
"Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3Bucket4E517469"
631+
}
632+
]
633+
]
634+
}
635+
]
636+
}
637+
],
638+
"Version": "2012-10-17"
639+
},
640+
"PolicyName": "ShellJob39ServiceRoleDefaultPolicy38A33919",
641+
"Roles": [
642+
{
643+
"Ref": "ShellJob39ServiceRole2F6F3768"
644+
}
645+
]
646+
}
647+
},
648+
"ShellJob390C141361": {
649+
"Type": "AWS::Glue::Job",
650+
"Properties": {
651+
"Command": {
652+
"Name": "pythonshell",
653+
"PythonVersion": "3.9",
654+
"ScriptLocation": {
655+
"Fn::Join": [
656+
"",
657+
[
658+
"s3://",
659+
{
660+
"Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3Bucket4E517469"
661+
},
662+
"/",
663+
{
664+
"Fn::Select": [
665+
0,
666+
{
667+
"Fn::Split": [
668+
"||",
669+
{
670+
"Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3VersionKeyF7753763"
671+
}
672+
]
673+
}
674+
]
675+
},
676+
{
677+
"Fn::Select": [
678+
1,
679+
{
680+
"Fn::Split": [
681+
"||",
682+
{
683+
"Ref": "AssetParameters432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855S3VersionKeyF7753763"
684+
}
685+
]
686+
}
687+
]
688+
}
689+
]
690+
]
691+
}
692+
},
693+
"Role": {
694+
"Fn::GetAtt": [
695+
"ShellJob39ServiceRole2F6F3768",
696+
"Arn"
697+
]
698+
},
699+
"DefaultArguments": {
700+
"--job-language": "python",
701+
"arg1": "value1",
702+
"arg2": "value2"
703+
},
704+
"GlueVersion": "1.0",
705+
"Name": "ShellJob39",
706+
"Tags": {
707+
"key": "value"
708+
}
709+
}
559710
}
560711
},
561712
"Parameters": {

‎packages/@aws-cdk/aws-glue/test/job.integ.snapshot/cdk.out

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"version":"17.0.0"}
1+
{"version":"20.0.0"}

‎packages/@aws-cdk/aws-glue/test/job.integ.snapshot/manifest.json

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "17.0.0",
2+
"version": "20.0.0",
33
"artifacts": {
44
"Tree": {
55
"type": "cdk:tree",
@@ -112,6 +112,24 @@
112112
"type": "aws:cdk:logicalId",
113113
"data": "ShellJob42E81F95"
114114
}
115+
],
116+
"/aws-glue-job/ShellJob39/ServiceRole/Resource": [
117+
{
118+
"type": "aws:cdk:logicalId",
119+
"data": "ShellJob39ServiceRole2F6F3768"
120+
}
121+
],
122+
"/aws-glue-job/ShellJob39/ServiceRole/DefaultPolicy/Resource": [
123+
{
124+
"type": "aws:cdk:logicalId",
125+
"data": "ShellJob39ServiceRoleDefaultPolicy38A33919"
126+
}
127+
],
128+
"/aws-glue-job/ShellJob39/Resource": [
129+
{
130+
"type": "aws:cdk:logicalId",
131+
"data": "ShellJob390C141361"
132+
}
115133
]
116134
},
117135
"displayName": "aws-glue-job"

0 commit comments

Comments
 (0)