Skip to content

Commit 437c39c

Browse files
mufaddal-rohawalamufaddal-rohawala
authored
Merge pull request #7 from aws/master
2 parents 4bc1d8d + 4fe5ea4 commit 437c39c

15 files changed

+671
-30
lines changed

‎CHANGELOG.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,22 @@
11
# Changelog
22

3+
## v2.61.0 (2021-10-11)
4+
5+
### Features
6+
7+
* add support for PyTorch 1.9.0
8+
9+
### Bug Fixes and Other Changes
10+
11+
* Update TRAINING_DEFAULT_TIMEOUT_MINUTES to 40 min
12+
* notebook test for parallel PRs
13+
14+
## v2.60.0 (2021-10-08)
15+
16+
### Features
17+
18+
* Add support for Hugging Face 4.10.2
19+
320
## v2.59.8 (2021-10-07)
421

522
### Bug Fixes and Other Changes

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.59.9.dev0
1+
2.61.1.dev0

‎src/sagemaker/fw_utils.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@
6060
)
6161
SM_DATAPARALLEL_SUPPORTED_FRAMEWORK_VERSIONS = {
6262
"tensorflow": ["2.3", "2.3.1", "2.3.2", "2.4", "2.4.1"],
63-
"pytorch": ["1.6", "1.6.0", "1.7", "1.7.1", "1.8", "1.8.0", "1.8.1"],
63+
"pytorch": ["1.6", "1.6.0", "1.7", "1.7.1", "1.8", "1.8.0", "1.8.1", "1.9", "1.9.0"],
6464
}
6565
SMDISTRIBUTED_SUPPORTED_STRATEGIES = ["dataparallel", "modelparallel"]
6666

@@ -298,7 +298,7 @@ def framework_name_from_image(image_uri):
298298
(tensorflow|mxnet|chainer|pytorch|scikit-learn|xgboost
299299
|huggingface-tensorflow|huggingface-pytorch)(?:-)?
300300
(scriptmode|training)?
301-
:(.*)-(.*?)-(py2|py3[67]?)(?:.*)$""",
301+
:(.*)-(.*?)-(py2|py3\d*)(?:.*)$""",
302302
re.VERBOSE,
303303
)
304304
name_match = name_pattern.match(sagemaker_match.group(9))
@@ -329,7 +329,7 @@ def framework_version_from_tag(image_tag):
329329
Returns:
330330
str: The framework version.
331331
"""
332-
tag_pattern = re.compile("^(.*)-(cpu|gpu)-(py2|py3[67]?)$")
332+
tag_pattern = re.compile(r"^(.*)-(cpu|gpu)-(py2|py3\d*)$")
333333
tag_match = tag_pattern.match(image_tag)
334334
return None if tag_match is None else tag_match.group(1)
335335

‎src/sagemaker/huggingface/estimator.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@ def __init__(
178178
if "enable_sagemaker_metrics" not in kwargs:
179179
kwargs["enable_sagemaker_metrics"] = True
180180

181+
kwargs["py_version"] = self.py_version
182+
181183
super(HuggingFace, self).__init__(
182184
entry_point, source_dir, hyperparameters, image_uri=image_uri, **kwargs
183185
)

‎src/sagemaker/image_uri_config/huggingface.json

Lines changed: 419 additions & 2 deletions
Large diffs are not rendered by default.

‎src/sagemaker/image_uri_config/pytorch.json

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@
6262
"1.5": "1.5.0",
6363
"1.6": "1.6.0",
6464
"1.7": "1.7.1",
65-
"1.8": "1.8.1"
65+
"1.8": "1.8.1",
66+
"1.9": "1.9.0"
6667
},
6768
"versions": {
6869
"0.4.0": {
@@ -433,6 +434,39 @@
433434
"us-west-2": "763104351884"
434435
},
435436
"repository": "pytorch-inference"
437+
},
438+
"1.9.0": {
439+
"py_versions": [
440+
"py38"
441+
],
442+
"registries": {
443+
"af-south-1": "626614931356",
444+
"ap-east-1": "871362719292",
445+
"ap-northeast-1": "763104351884",
446+
"ap-northeast-2": "763104351884",
447+
"ap-northeast-3": "364406365360",
448+
"ap-south-1": "763104351884",
449+
"ap-southeast-1": "763104351884",
450+
"ap-southeast-2": "763104351884",
451+
"ca-central-1": "763104351884",
452+
"cn-north-1": "727897471807",
453+
"cn-northwest-1": "727897471807",
454+
"eu-central-1": "763104351884",
455+
"eu-north-1": "763104351884",
456+
"eu-west-1": "763104351884",
457+
"eu-west-2": "763104351884",
458+
"eu-west-3": "763104351884",
459+
"eu-south-1": "692866216735",
460+
"me-south-1": "217643126080",
461+
"sa-east-1": "763104351884",
462+
"us-east-1": "763104351884",
463+
"us-east-2": "763104351884",
464+
"us-gov-west-1": "442386744353",
465+
"us-iso-east-1": "886529160074",
466+
"us-west-1": "763104351884",
467+
"us-west-2": "763104351884"
468+
},
469+
"repository": "pytorch-inference"
436470
}
437471
}
438472
},
@@ -451,7 +485,8 @@
451485
"1.5": "1.5.0",
452486
"1.6": "1.6.0",
453487
"1.7": "1.7.1",
454-
"1.8": "1.8.1"
488+
"1.8": "1.8.1",
489+
"1.9": "1.9.0"
455490
},
456491
"versions": {
457492
"0.4.0": {
@@ -823,6 +858,39 @@
823858
"us-west-2": "763104351884"
824859
},
825860
"repository": "pytorch-training"
861+
},
862+
"1.9.0": {
863+
"py_versions": [
864+
"py38"
865+
],
866+
"registries": {
867+
"af-south-1": "626614931356",
868+
"ap-east-1": "871362719292",
869+
"ap-northeast-1": "763104351884",
870+
"ap-northeast-2": "763104351884",
871+
"ap-northeast-3": "364406365360",
872+
"ap-south-1": "763104351884",
873+
"ap-southeast-1": "763104351884",
874+
"ap-southeast-2": "763104351884",
875+
"ca-central-1": "763104351884",
876+
"cn-north-1": "727897471807",
877+
"cn-northwest-1": "727897471807",
878+
"eu-central-1": "763104351884",
879+
"eu-north-1": "763104351884",
880+
"eu-west-1": "763104351884",
881+
"eu-west-2": "763104351884",
882+
"eu-west-3": "763104351884",
883+
"eu-south-1": "692866216735",
884+
"me-south-1": "217643126080",
885+
"sa-east-1": "763104351884",
886+
"us-east-1": "763104351884",
887+
"us-east-2": "763104351884",
888+
"us-gov-west-1": "442386744353",
889+
"us-iso-east-1": "886529160074",
890+
"us-west-1": "763104351884",
891+
"us-west-2": "763104351884"
892+
},
893+
"repository": "pytorch-training"
826894
}
827895
}
828896
}

‎tests/conftest.py

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,8 @@ def mxnet_eia_latest_py_version():
184184
def pytorch_training_py_version(pytorch_training_version, request):
185185
if Version(pytorch_training_version) < Version("1.5.0"):
186186
return request.param
187-
elif Version(pytorch_training_version) == Version("1.7.1"):
188-
return "py36"
187+
elif Version(pytorch_training_version) >= Version("1.9"):
188+
return "py38"
189189
else:
190190
return "py3"
191191

@@ -194,8 +194,8 @@ def pytorch_training_py_version(pytorch_training_version, request):
194194
def pytorch_inference_py_version(pytorch_inference_version, request):
195195
if Version(pytorch_inference_version) < Version("1.4.0"):
196196
return request.param
197-
elif Version(pytorch_inference_version) == Version("1.7.1"):
198-
return "py36"
197+
elif Version(pytorch_inference_version) >= Version("1.9"):
198+
return "py38"
199199
else:
200200
return "py3"
201201

@@ -207,6 +207,32 @@ def huggingface_pytorch_training_version(huggingface_training_version):
207207
)[0]
208208

209209

210+
@pytest.fixture(scope="module")
211+
def huggingface_pytorch_training_py_version(huggingface_pytorch_training_version):
212+
return "py38" if Version(huggingface_pytorch_training_version) >= Version("1.9") else "py36"
213+
214+
215+
@pytest.fixture(scope="module")
216+
def huggingface_pytorch_latest_training_py_version(huggingface_training_pytorch_latest_version):
217+
return (
218+
"py38" if Version(huggingface_training_pytorch_latest_version) >= Version("1.9") else "py36"
219+
)
220+
221+
222+
@pytest.fixture(scope="module")
223+
def huggingface_pytorch_latest_inference_py_version(huggingface_inference_pytorch_latest_version):
224+
return (
225+
"py38"
226+
if Version(huggingface_inference_pytorch_latest_version) >= Version("1.9")
227+
else "py36"
228+
)
229+
230+
231+
@pytest.fixture(scope="module")
232+
def huggingface_tensorflow_latest_training_py_version():
233+
return "py37"
234+
235+
210236
@pytest.fixture(scope="module")
211237
def pytorch_eia_py_version():
212238
return "py3"

‎tests/integ/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import boto3
1919

2020
DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "data")
21-
TRAINING_DEFAULT_TIMEOUT_MINUTES = 20
21+
TRAINING_DEFAULT_TIMEOUT_MINUTES = 40
2222
TUNING_DEFAULT_TIMEOUT_MINUTES = 20
2323
TRANSFORM_DEFAULT_TIMEOUT_MINUTES = 20
2424
AUTO_ML_DEFAULT_TIMEMOUT_MINUTES = 60

‎tests/integ/test_huggingface.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ def test_framework_processing_job_with_deps(
3636
gpu_instance_type,
3737
huggingface_training_latest_version,
3838
huggingface_training_pytorch_latest_version,
39+
huggingface_pytorch_latest_training_py_version,
3940
):
4041
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
4142
code_path = os.path.join(DATA_DIR, "dummy_code_bundle_with_reqs")
@@ -44,7 +45,7 @@ def test_framework_processing_job_with_deps(
4445
processor = HuggingFaceProcessor(
4546
transformers_version=huggingface_training_latest_version,
4647
pytorch_version=huggingface_training_pytorch_latest_version,
47-
py_version="py36",
48+
py_version=huggingface_pytorch_latest_training_py_version,
4849
role=ROLE,
4950
instance_count=1,
5051
instance_type=gpu_instance_type,
@@ -70,12 +71,13 @@ def test_huggingface_training(
7071
gpu_instance_type,
7172
huggingface_training_latest_version,
7273
huggingface_training_pytorch_latest_version,
74+
huggingface_pytorch_latest_training_py_version,
7375
):
7476
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
7577
data_path = os.path.join(DATA_DIR, "huggingface")
7678

7779
hf = HuggingFace(
78-
py_version="py36",
80+
py_version=huggingface_pytorch_latest_training_py_version,
7981
entry_point=os.path.join(data_path, "run_glue.py"),
8082
role="SageMakerRole",
8183
transformers_version=huggingface_training_latest_version,
@@ -113,12 +115,13 @@ def test_huggingface_training_tf(
113115
gpu_instance_type,
114116
huggingface_training_latest_version,
115117
huggingface_training_tensorflow_latest_version,
118+
huggingface_tensorflow_latest_training_py_version,
116119
):
117120
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
118121
data_path = os.path.join(DATA_DIR, "huggingface")
119122

120123
hf = HuggingFace(
121-
py_version="py37",
124+
py_version=huggingface_tensorflow_latest_training_py_version,
122125
entry_point=os.path.join(data_path, "run_tf.py"),
123126
role=ROLE,
124127
transformers_version=huggingface_training_latest_version,
@@ -152,6 +155,7 @@ def test_huggingface_inference(
152155
gpu_instance_type,
153156
huggingface_inference_latest_version,
154157
huggingface_inference_pytorch_latest_version,
158+
huggingface_pytorch_latest_inference_py_version,
155159
):
156160
env = {
157161
"HF_MODEL_ID": "sshleifer/tiny-distilbert-base-uncased-finetuned-sst-2-english",
@@ -163,7 +167,7 @@ def test_huggingface_inference(
163167
sagemaker_session=sagemaker_session,
164168
role="SageMakerRole",
165169
env=env,
166-
py_version="py36",
170+
py_version=huggingface_pytorch_latest_inference_py_version,
167171
transformers_version=huggingface_inference_latest_version,
168172
pytorch_version=huggingface_inference_pytorch_latest_version,
169173
)

0 commit comments

Comments
 (0)