Skip to content

Commit dba8cf3

Browse files
authored
fix(glue): synth time validation does not work in Python/Java/C#/Go (#26650)
Fixes some issues on Glue Job synth time validation: - `maxCapacity` can be specified for PythonShell jobs with Glue Version 2.0+ and should be either 0.0625 or 1 (defaults to 0.0625) - Validation was not working when using `GlueVersion.of()` and `JobType.of()` - PythonShell jobs supports GlueVersion 3.0 Closes #26620. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 972a06f commit dba8cf3

14 files changed

+1239
-17
lines changed

packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts

+15-13
Original file line numberDiff line numberDiff line change
@@ -350,38 +350,40 @@ export class JobExecutable {
350350
private config: JobExecutableConfig;
351351

352352
private constructor(config: JobExecutableConfig) {
353-
if (JobType.PYTHON_SHELL === config.type) {
353+
const glueVersion = config.glueVersion.name;
354+
const type = config.type.name;
355+
if (JobType.PYTHON_SHELL.name === type) {
354356
if (config.language !== JobLanguage.PYTHON) {
355357
throw new Error('Python shell requires the language to be set to Python');
356358
}
357-
if ([GlueVersion.V0_9, GlueVersion.V3_0, GlueVersion.V4_0].includes(config.glueVersion)) {
358-
throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Python Shell`);
359+
if ([GlueVersion.V0_9.name, GlueVersion.V4_0.name].includes(glueVersion)) {
360+
throw new Error(`Specified GlueVersion ${glueVersion} does not support Python Shell`);
359361
}
360362
}
361-
if (JobType.RAY === config.type) {
363+
if (JobType.RAY.name === type) {
362364
if (config.language !== JobLanguage.PYTHON) {
363365
throw new Error('Ray requires the language to be set to Python');
364366
}
365-
if ([GlueVersion.V0_9, GlueVersion.V1_0, GlueVersion.V2_0, GlueVersion.V3_0].includes(config.glueVersion)) {
366-
throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Ray`);
367+
if ([GlueVersion.V0_9.name, GlueVersion.V1_0.name, GlueVersion.V2_0.name, GlueVersion.V3_0.name].includes(glueVersion)) {
368+
throw new Error(`Specified GlueVersion ${glueVersion} does not support Ray`);
367369
}
368370
}
369-
if (config.extraJarsFirst && [GlueVersion.V0_9, GlueVersion.V1_0].includes(config.glueVersion)) {
370-
throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support extraJarsFirst`);
371+
if (config.extraJarsFirst && [GlueVersion.V0_9.name, GlueVersion.V1_0.name].includes(glueVersion)) {
372+
throw new Error(`Specified GlueVersion ${glueVersion} does not support extraJarsFirst`);
371373
}
372-
if (config.pythonVersion === PythonVersion.TWO && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(config.glueVersion)) {
373-
throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support PythonVersion ${config.pythonVersion}`);
374+
if (config.pythonVersion === PythonVersion.TWO && ![GlueVersion.V0_9.name, GlueVersion.V1_0.name].includes(glueVersion)) {
375+
throw new Error(`Specified GlueVersion ${glueVersion} does not support PythonVersion ${config.pythonVersion}`);
374376
}
375377
if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) {
376378
throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON');
377379
}
378-
if (config.pythonVersion === PythonVersion.THREE_NINE && config.type !== JobType.PYTHON_SHELL && config.type !== JobType.RAY) {
380+
if (config.pythonVersion === PythonVersion.THREE_NINE && type !== JobType.PYTHON_SHELL.name && type !== JobType.RAY.name) {
379381
throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray');
380382
}
381-
if (config.pythonVersion === PythonVersion.THREE && config.type === JobType.RAY) {
383+
if (config.pythonVersion === PythonVersion.THREE && type === JobType.RAY.name) {
382384
throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray');
383385
}
384-
if (config.runtime === undefined && config.type === JobType.RAY) {
386+
if (config.runtime === undefined && type === JobType.RAY.name) {
385387
throw new Error('Runtime is required for Ray jobs.');
386388
}
387389
this.config = config;

packages/@aws-cdk/aws-glue-alpha/lib/job.ts

+12-3
Original file line numberDiff line numberDiff line change
@@ -719,11 +719,20 @@ export class Job extends JobBase {
719719
}
720720
}
721721

722-
if (props.maxCapacity !== undefined && (props.workerType && props.workerCount !== undefined)) {
722+
let maxCapacity = props.maxCapacity;
723+
if (maxCapacity !== undefined && (props.workerType && props.workerCount !== undefined)) {
723724
throw new Error('maxCapacity cannot be used when setting workerType and workerCount');
724725
}
725-
if (props.maxCapacity !== undefined && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(executable.glueVersion)) {
726-
throw new Error('maxCapacity cannot be used when GlueVersion 2.0 or later');
726+
if (executable.type !== JobType.PYTHON_SHELL) {
727+
if (maxCapacity !== undefined && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(executable.glueVersion)) {
728+
throw new Error('maxCapacity cannot be used when GlueVersion 2.0 or later');
729+
}
730+
} else {
731+
// max capacity validation for python shell jobs (defaults to 0.0625)
732+
maxCapacity = maxCapacity ?? 0.0625;
733+
if (maxCapacity !== 0.0625 && maxCapacity !== 1) {
734+
throw new Error(`maxCapacity value must be either 0.0625 or 1 for JobType.PYTHON_SHELL jobs, received ${maxCapacity}`);
735+
}
727736
}
728737
if ((!props.workerType && props.workerCount !== undefined) || (props.workerType && props.workerCount === undefined)) {
729738
throw new Error('Both workerType and workerCount must be set');
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
print("hello world")
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"version": "33.0.0",
3+
"files": {
4+
"432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": {
5+
"source": {
6+
"path": "asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py",
7+
"packaging": "file"
8+
},
9+
"destinations": {
10+
"current_account-current_region": {
11+
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
12+
"objectKey": "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py",
13+
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
14+
}
15+
}
16+
},
17+
"13432a74ca6cfada399f4d2b33385964f66c49aeeb01c5f0cefec52560a4dffa": {
18+
"source": {
19+
"path": "aws-glue-job-python-shell.template.json",
20+
"packaging": "file"
21+
},
22+
"destinations": {
23+
"current_account-current_region": {
24+
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
25+
"objectKey": "13432a74ca6cfada399f4d2b33385964f66c49aeeb01c5f0cefec52560a4dffa.json",
26+
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
27+
}
28+
}
29+
}
30+
},
31+
"dockerImages": {}
32+
}

0 commit comments

Comments
 (0)