Skip to content

Commit 5fef403

Browse files
tmyodatmyoda
authored
feat(stepfunctions-tasks): FastFile mode for SageMaker Training Job (#26675)
According to the AWS documentation, the TrainingInputMode for a SageMaker Training Job can be one of the following: `Pipe | File | FastFile` https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_Channel.html#sagemaker-Type-Channel-InputMode https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html#API_AlgorithmSpecification_Contents I have just added `FastFile` below to align with the official documentation. https://github.com/aws/aws-cdk/blob/v2.90.0/packages/aws-cdk-lib/aws-stepfunctions-tasks/lib/sagemaker/base-types.ts#L458 Closes #26653. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 7e1c038 commit 5fef403

File tree

10 files changed

+61
-49
lines changed

10 files changed

+61
-49
lines changed

‎packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/sagemaker/integ.create-training-job.js.snapshot/cdk.out

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"version":"30.0.0"}
1+
{"version":"33.0.0"}

‎packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/sagemaker/integ.create-training-job.js.snapshot/integ-stepfunctions-sagemaker.assets.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
2-
"version": "30.0.0",
2+
"version": "33.0.0",
33
"files": {
4-
"81a9a9ef07dfc045ad84514bf0771e76c20386202c7c26130f4f954b8411de26": {
4+
"11bc70dd89e4afb387b1b639ea823eb492c0e7a7dd664e50aa483f10ef3e204a": {
55
"source": {
66
"path": "integ-stepfunctions-sagemaker.template.json",
77
"packaging": "file"
88
},
99
"destinations": {
1010
"current_account-current_region": {
1111
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
12-
"objectKey": "81a9a9ef07dfc045ad84514bf0771e76c20386202c7c26130f4f954b8411de26.json",
12+
"objectKey": "11bc70dd89e4afb387b1b639ea823eb492c0e7a7dd664e50aa483f10ef3e204a.json",
1313
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
1414
}
1515
}

‎packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/sagemaker/integ.create-training-job.js.snapshot/integ-stepfunctions-sagemaker.template.json

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -276,12 +276,6 @@
276276
"StateMachine2E01A3A5": {
277277
"Type": "AWS::StepFunctions::StateMachine",
278278
"Properties": {
279-
"RoleArn": {
280-
"Fn::GetAtt": [
281-
"StateMachineRoleB840431D",
282-
"Arn"
283-
]
284-
},
285279
"DefinitionString": {
286280
"Fn::Join": [
287281
"",
@@ -297,7 +291,7 @@
297291
"Arn"
298292
]
299293
},
300-
"\",\"AlgorithmSpecification\":{\"TrainingInputMode\":\"File\",\"AlgorithmName\":\"arn:aws:sagemaker:us-east-1:865070037744:algorithm/scikit-decision-trees-15423055-57b73412d2e93e9239e4e16f83298b8f\"},\"InputDataConfig\":[{\"ChannelName\":\"InputData\",\"DataSource\":{\"S3DataSource\":{\"S3Uri\":\"https://s3.",
294+
"\",\"AlgorithmSpecification\":{\"TrainingInputMode\":\"FastFile\",\"AlgorithmName\":\"BlazingText\"},\"InputDataConfig\":[{\"ChannelName\":\"InputData\",\"DataSource\":{\"S3DataSource\":{\"S3Uri\":\"https://s3.",
301295
{
302296
"Ref": "AWS::Region"
303297
},
@@ -324,6 +318,12 @@
324318
"/result/\"},\"ResourceConfig\":{\"InstanceCount\":1,\"InstanceType\":\"ml.m4.xlarge\",\"VolumeSizeInGB\":10},\"StoppingCondition\":{\"MaxRuntimeInSeconds\":3600}}}}}"
325319
]
326320
]
321+
},
322+
"RoleArn": {
323+
"Fn::GetAtt": [
324+
"StateMachineRoleB840431D",
325+
"Arn"
326+
]
327327
}
328328
},
329329
"DependsOn": [

‎packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/sagemaker/integ.create-training-job.js.snapshot/integ.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "30.0.0",
2+
"version": "33.0.0",
33
"testCases": {
44
"integ.create-training-job": {
55
"stacks": [

‎packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/sagemaker/integ.create-training-job.js.snapshot/manifest.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "30.0.0",
2+
"version": "33.0.0",
33
"artifacts": {
44
"integ-stepfunctions-sagemaker.assets": {
55
"type": "cdk:asset-manifest",
@@ -17,7 +17,7 @@
1717
"validateOnSynth": false,
1818
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
1919
"cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
20-
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/81a9a9ef07dfc045ad84514bf0771e76c20386202c7c26130f4f954b8411de26.json",
20+
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/11bc70dd89e4afb387b1b639ea823eb492c0e7a7dd664e50aa483f10ef3e204a.json",
2121
"requiresBootstrapStackVersion": 6,
2222
"bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
2323
"additionalDependencies": [

‎packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/sagemaker/integ.create-training-job.js.snapshot/tree.json

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -49,13 +49,13 @@
4949
}
5050
},
5151
"constructInfo": {
52-
"fqn": "@aws-cdk/aws-kms.CfnKey",
52+
"fqn": "aws-cdk-lib.aws_kms.CfnKey",
5353
"version": "0.0.0"
5454
}
5555
}
5656
},
5757
"constructInfo": {
58-
"fqn": "@aws-cdk/aws-kms.Key",
58+
"fqn": "aws-cdk-lib.aws_kms.Key",
5959
"version": "0.0.0"
6060
}
6161
},
@@ -87,13 +87,13 @@
8787
}
8888
},
8989
"constructInfo": {
90-
"fqn": "@aws-cdk/aws-s3.CfnBucket",
90+
"fqn": "aws-cdk-lib.aws_s3.CfnBucket",
9191
"version": "0.0.0"
9292
}
9393
}
9494
},
9595
"constructInfo": {
96-
"fqn": "@aws-cdk/aws-s3.Bucket",
96+
"fqn": "aws-cdk-lib.aws_s3.Bucket",
9797
"version": "0.0.0"
9898
}
9999
},
@@ -109,7 +109,7 @@
109109
"id": "ImportSagemakerRole",
110110
"path": "integ-stepfunctions-sagemaker/TrainTask/SagemakerRole/ImportSagemakerRole",
111111
"constructInfo": {
112-
"fqn": "@aws-cdk/core.Resource",
112+
"fqn": "aws-cdk-lib.Resource",
113113
"version": "0.0.0"
114114
}
115115
},
@@ -156,7 +156,7 @@
156156
}
157157
},
158158
"constructInfo": {
159-
"fqn": "@aws-cdk/aws-iam.CfnRole",
159+
"fqn": "aws-cdk-lib.aws_iam.CfnRole",
160160
"version": "0.0.0"
161161
}
162162
},
@@ -264,25 +264,25 @@
264264
}
265265
},
266266
"constructInfo": {
267-
"fqn": "@aws-cdk/aws-iam.CfnPolicy",
267+
"fqn": "aws-cdk-lib.aws_iam.CfnPolicy",
268268
"version": "0.0.0"
269269
}
270270
}
271271
},
272272
"constructInfo": {
273-
"fqn": "@aws-cdk/aws-iam.Policy",
273+
"fqn": "aws-cdk-lib.aws_iam.Policy",
274274
"version": "0.0.0"
275275
}
276276
}
277277
},
278278
"constructInfo": {
279-
"fqn": "@aws-cdk/aws-iam.Role",
279+
"fqn": "aws-cdk-lib.aws_iam.Role",
280280
"version": "0.0.0"
281281
}
282282
}
283283
},
284284
"constructInfo": {
285-
"fqn": "@aws-cdk/aws-stepfunctions-tasks.SageMakerCreateTrainingJob",
285+
"fqn": "aws-cdk-lib.aws_stepfunctions_tasks.SageMakerCreateTrainingJob",
286286
"version": "0.0.0"
287287
}
288288
},
@@ -298,7 +298,7 @@
298298
"id": "ImportRole",
299299
"path": "integ-stepfunctions-sagemaker/StateMachine/Role/ImportRole",
300300
"constructInfo": {
301-
"fqn": "@aws-cdk/core.Resource",
301+
"fqn": "aws-cdk-lib.Resource",
302302
"version": "0.0.0"
303303
}
304304
},
@@ -323,7 +323,7 @@
323323
}
324324
},
325325
"constructInfo": {
326-
"fqn": "@aws-cdk/aws-iam.CfnRole",
326+
"fqn": "aws-cdk-lib.aws_iam.CfnRole",
327327
"version": "0.0.0"
328328
}
329329
},
@@ -399,19 +399,19 @@
399399
}
400400
},
401401
"constructInfo": {
402-
"fqn": "@aws-cdk/aws-iam.CfnPolicy",
402+
"fqn": "aws-cdk-lib.aws_iam.CfnPolicy",
403403
"version": "0.0.0"
404404
}
405405
}
406406
},
407407
"constructInfo": {
408-
"fqn": "@aws-cdk/aws-iam.Policy",
408+
"fqn": "aws-cdk-lib.aws_iam.Policy",
409409
"version": "0.0.0"
410410
}
411411
}
412412
},
413413
"constructInfo": {
414-
"fqn": "@aws-cdk/aws-iam.Role",
414+
"fqn": "aws-cdk-lib.aws_iam.Role",
415415
"version": "0.0.0"
416416
}
417417
},
@@ -421,12 +421,6 @@
421421
"attributes": {
422422
"aws:cdk:cloudformation:type": "AWS::StepFunctions::StateMachine",
423423
"aws:cdk:cloudformation:props": {
424-
"roleArn": {
425-
"Fn::GetAtt": [
426-
"StateMachineRoleB840431D",
427-
"Arn"
428-
]
429-
},
430424
"definitionString": {
431425
"Fn::Join": [
432426
"",
@@ -442,7 +436,7 @@
442436
"Arn"
443437
]
444438
},
445-
"\",\"AlgorithmSpecification\":{\"TrainingInputMode\":\"File\",\"AlgorithmName\":\"arn:aws:sagemaker:us-east-1:865070037744:algorithm/scikit-decision-trees-15423055-57b73412d2e93e9239e4e16f83298b8f\"},\"InputDataConfig\":[{\"ChannelName\":\"InputData\",\"DataSource\":{\"S3DataSource\":{\"S3Uri\":\"https://s3.",
439+
"\",\"AlgorithmSpecification\":{\"TrainingInputMode\":\"FastFile\",\"AlgorithmName\":\"BlazingText\"},\"InputDataConfig\":[{\"ChannelName\":\"InputData\",\"DataSource\":{\"S3DataSource\":{\"S3Uri\":\"https://s3.",
446440
{
447441
"Ref": "AWS::Region"
448442
},
@@ -469,47 +463,53 @@
469463
"/result/\"},\"ResourceConfig\":{\"InstanceCount\":1,\"InstanceType\":\"ml.m4.xlarge\",\"VolumeSizeInGB\":10},\"StoppingCondition\":{\"MaxRuntimeInSeconds\":3600}}}}}"
470464
]
471465
]
466+
},
467+
"roleArn": {
468+
"Fn::GetAtt": [
469+
"StateMachineRoleB840431D",
470+
"Arn"
471+
]
472472
}
473473
}
474474
},
475475
"constructInfo": {
476-
"fqn": "@aws-cdk/aws-stepfunctions.CfnStateMachine",
476+
"fqn": "aws-cdk-lib.aws_stepfunctions.CfnStateMachine",
477477
"version": "0.0.0"
478478
}
479479
}
480480
},
481481
"constructInfo": {
482-
"fqn": "@aws-cdk/aws-stepfunctions.StateMachine",
482+
"fqn": "aws-cdk-lib.aws_stepfunctions.StateMachine",
483483
"version": "0.0.0"
484484
}
485485
},
486486
"stateMachineArn": {
487487
"id": "stateMachineArn",
488488
"path": "integ-stepfunctions-sagemaker/stateMachineArn",
489489
"constructInfo": {
490-
"fqn": "@aws-cdk/core.CfnOutput",
490+
"fqn": "aws-cdk-lib.CfnOutput",
491491
"version": "0.0.0"
492492
}
493493
},
494494
"BootstrapVersion": {
495495
"id": "BootstrapVersion",
496496
"path": "integ-stepfunctions-sagemaker/BootstrapVersion",
497497
"constructInfo": {
498-
"fqn": "@aws-cdk/core.CfnParameter",
498+
"fqn": "aws-cdk-lib.CfnParameter",
499499
"version": "0.0.0"
500500
}
501501
},
502502
"CheckBootstrapVersion": {
503503
"id": "CheckBootstrapVersion",
504504
"path": "integ-stepfunctions-sagemaker/CheckBootstrapVersion",
505505
"constructInfo": {
506-
"fqn": "@aws-cdk/core.CfnRule",
506+
"fqn": "aws-cdk-lib.CfnRule",
507507
"version": "0.0.0"
508508
}
509509
}
510510
},
511511
"constructInfo": {
512-
"fqn": "@aws-cdk/core.Stack",
512+
"fqn": "aws-cdk-lib.Stack",
513513
"version": "0.0.0"
514514
}
515515
},
@@ -518,12 +518,12 @@
518518
"path": "Tree",
519519
"constructInfo": {
520520
"fqn": "constructs.Construct",
521-
"version": "10.1.237"
521+
"version": "10.2.69"
522522
}
523523
}
524524
},
525525
"constructInfo": {
526-
"fqn": "@aws-cdk/core.App",
526+
"fqn": "aws-cdk-lib.App",
527527
"version": "0.0.0"
528528
}
529529
}

‎packages/@aws-cdk-testing/framework-integ/test/aws-stepfunctions-tasks/test/sagemaker/integ.create-training-job.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { Key } from 'aws-cdk-lib/aws-kms';
22
import { Bucket, BucketEncryption } from 'aws-cdk-lib/aws-s3';
33
import { StateMachine } from 'aws-cdk-lib/aws-stepfunctions';
44
import { App, CfnOutput, RemovalPolicy, Stack } from 'aws-cdk-lib';
5-
import { S3Location, SageMakerCreateTrainingJob } from 'aws-cdk-lib/aws-stepfunctions-tasks';
5+
import { S3Location, SageMakerCreateTrainingJob, InputMode } from 'aws-cdk-lib/aws-stepfunctions-tasks';
66

77
/*
88
* Creates a state machine with a task state to create a training job in AWS SageMaker
@@ -35,7 +35,8 @@ const trainingData = new Bucket(stack, 'TrainingData', {
3535
const sm = new StateMachine(stack, 'StateMachine', {
3636
definition: new SageMakerCreateTrainingJob(stack, 'TrainTask', {
3737
algorithmSpecification: {
38-
algorithmName: 'arn:aws:sagemaker:us-east-1:865070037744:algorithm/scikit-decision-trees-15423055-57b73412d2e93e9239e4e16f83298b8f',
38+
algorithmName: 'BlazingText',
39+
trainingInputMode: InputMode.FAST_FILE,
3940
},
4041
inputDataConfig: [{
4142
channelName: 'InputData',

‎packages/aws-cdk-lib/aws-stepfunctions-tasks/README.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1060,6 +1060,12 @@ new tasks.SageMakerCreateTrainingJob(this, 'TrainSagemaker', {
10601060
});
10611061
```
10621062

1063+
You can specify [TrainingInputMode](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html#API_AlgorithmSpecification_Contents) via the trainingInputMode property.
1064+
1065+
- To download the data from Amazon Simple Storage Service (Amazon S3) to the provisioned ML storage volume, and mount the directory to a Docker volume, choose `InputMode.FILE` if an algorithm supports it.
1066+
- To stream data directly from Amazon S3 to the container, choose `InputMode.PIPE` if an algorithm supports it.
1067+
- To stream data directly from Amazon S3 to the container with no code changes and to provide file system access to the data, choose `InputMode.FAST_FILE` if an algorithm supports it.
1068+
10631069
### Create Transform Job
10641070

10651071
You can call the [`CreateTransformJob`](https://docs.aws.amazon.com/sagemaker/latest/dg/API_CreateTransformJob.html) API from a `Task` state.

‎packages/aws-cdk-lib/aws-stepfunctions-tasks/lib/sagemaker/base-types.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,12 @@ export enum InputMode {
464464
/**
465465
* File mode.
466466
*/
467-
FILE = 'File'
467+
FILE = 'File',
468+
469+
/**
470+
* FastFile mode.
471+
*/
472+
FAST_FILE = 'FastFile'
468473
}
469474

470475
/**

‎packages/aws-cdk-lib/aws-stepfunctions-tasks/test/sagemaker/create-training-job.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ test('create complex training job', () => {
134134
role,
135135
algorithmSpecification: {
136136
algorithmName: 'BlazingText',
137-
trainingInputMode: tasks.InputMode.FILE,
137+
trainingInputMode: tasks.InputMode.FAST_FILE,
138138
metricDefinitions: [
139139
{
140140
name: 'mymetric', regex: 'regex_pattern',
@@ -218,7 +218,7 @@ test('create complex training job', () => {
218218
TrainingJobName: 'MyTrainJob',
219219
RoleArn: { 'Fn::GetAtt': ['Role1ABCC5F0', 'Arn'] },
220220
AlgorithmSpecification: {
221-
TrainingInputMode: 'File',
221+
TrainingInputMode: 'FastFile',
222222
AlgorithmName: 'BlazingText',
223223
MetricDefinitions: [
224224
{ Name: 'mymetric', Regex: 'regex_pattern' },

0 commit comments

Comments
 (0)