Skip to content

Commit 00d86ba

Browse files
jeniyatjeniyat
authored
Merge branch 'master' into feat/fw-processor-normargs
2 parents 02e0db1 + 529256d commit 00d86ba

21 files changed

+585
-38
lines changed

‎.github/ISSUE_TEMPLATE/bug_report.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: Bug report
33
about: File a report to help us reproduce and fix the problem
44
title: ''
5-
labels: ''
5+
labels: 'bug'
66
assignees: ''
77

88
---

‎.github/ISSUE_TEMPLATE/feature_request.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: Feature request
33
about: Suggest new functionality for this library
44
title: ''
5-
labels: ''
5+
labels: 'feature request'
66
assignees: ''
77

88
---

‎CHANGELOG.md

+38
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,43 @@
11
# Changelog
22

3+
## v2.60.0 (2021-10-08)
4+
5+
### Features
6+
7+
* Add support for Hugging Face 4.10.2
8+
9+
## v2.59.8 (2021-10-07)
10+
11+
### Bug Fixes and Other Changes
12+
13+
* fix feature store ingestion via data wrangler test
14+
15+
## v2.59.7 (2021-10-04)
16+
17+
### Bug Fixes and Other Changes
18+
19+
* update feature request label
20+
* update bug template
21+
22+
## v2.59.6 (2021-09-30)
23+
24+
### Bug Fixes and Other Changes
25+
26+
* ParamValidationError when scheduling a Clarify model monitor
27+
28+
## v2.59.5 (2021-09-29)
29+
30+
### Bug Fixes and Other Changes
31+
32+
* support maps in step parameters
33+
34+
## v2.59.4 (2021-09-27)
35+
36+
### Bug Fixes and Other Changes
37+
38+
* add checks for ExecutionRole in UserSettings, adds more unit tests
39+
* add pytorch 1.8.1 for huggingface
40+
341
## v2.59.3.post0 (2021-09-22)
442

543
### Documentation Changes

‎VERSION

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.59.4.dev0
1+
2.60.1.dev0

‎src/sagemaker/clarify.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ def __init__(
228228
probability_threshold=None,
229229
label_headers=None,
230230
):
231-
"""Initializes a model output config to extract the predicted label.
231+
"""Initializes a model output config to extract the predicted label or predicted score(s).
232232
233233
The following examples show different parameter configurations depending on the endpoint:
234234
* Regression Task: The model returns the score, e.g. 1.2. we don't need to specify
@@ -255,11 +255,11 @@ def __init__(
255255
'label_headers=['cat','dog','fish']' and infer the predicted label to be 'fish.'
256256
257257
Args:
258-
label (str or int or list[int]): Optional index or JSONPath location in the model
259-
output for the prediction. In case, this is a predicted label of the same type as
260-
the label in the dataset no further arguments need to be specified.
261-
probability (str or int or list[int]): Optional index or JSONPath location in the model
262-
output for the predicted scores.
258+
label (str or int): Index or JSONPath location in the model output for the prediction.
259+
In case, this is a predicted label of the same type as the label in the dataset,
260+
no further arguments need to be specified.
261+
probability (str or int): Index or JSONPath location in the model output
262+
for the predicted score(s).
263263
probability_threshold (float): An optional value for binary prediction tasks in which
264264
the model returns a probability, to indicate the threshold to convert the
265265
prediction to a boolean value. Default is 0.5.

‎src/sagemaker/huggingface/estimator.py

+2
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@ def __init__(
178178
if "enable_sagemaker_metrics" not in kwargs:
179179
kwargs["enable_sagemaker_metrics"] = True
180180

181+
kwargs["py_version"] = self.py_version
182+
181183
super(HuggingFace, self).__init__(
182184
entry_point, source_dir, hyperparameters, image_uri=image_uri, **kwargs
183185
)

‎src/sagemaker/image_uri_config/huggingface.json

+144-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
"version_aliases": {
55
"4.4": "4.4.2",
66
"4.5": "4.5.0",
7-
"4.6": "4.6.1"
7+
"4.6": "4.6.1",
8+
"4.10": "4.10.2"
89
},
910
"versions": {
1011
"4.4.2": {
@@ -278,14 +279,85 @@
278279
"repository": "huggingface-tensorflow-training",
279280
"container_version": {"gpu":"cu110-ubuntu18.04"}
280281
}
282+
},
283+
"4.10.2": {
284+
"version_aliases": {
285+
"pytorch1.9": "pytorch1.9.0",
286+
"tensorflow2.5": "tensorflow2.5.1"
287+
},
288+
"pytorch1.9.0": {
289+
"py_versions": ["py38"],
290+
"registries": {
291+
"af-south-1": "626614931356",
292+
"ap-east-1": "871362719292",
293+
"ap-northeast-1": "763104351884",
294+
"ap-northeast-2": "763104351884",
295+
"ap-northeast-3": "364406365360",
296+
"ap-south-1": "763104351884",
297+
"ap-southeast-1": "763104351884",
298+
"ap-southeast-2": "763104351884",
299+
"ca-central-1": "763104351884",
300+
"cn-north-1": "727897471807",
301+
"cn-northwest-1": "727897471807",
302+
"eu-central-1": "763104351884",
303+
"eu-north-1": "763104351884",
304+
"eu-west-1": "763104351884",
305+
"eu-west-2": "763104351884",
306+
"eu-west-3": "763104351884",
307+
"eu-south-1": "692866216735",
308+
"me-south-1": "217643126080",
309+
"sa-east-1": "763104351884",
310+
"us-east-1": "763104351884",
311+
"us-east-2": "763104351884",
312+
"us-gov-west-1": "442386744353",
313+
"us-iso-east-1": "886529160074",
314+
"us-west-1": "763104351884",
315+
"us-west-2": "763104351884"
316+
},
317+
"repository": "huggingface-pytorch-training",
318+
"container_version": {"gpu": "cu111-ubuntu20.04"}
319+
},
320+
"tensorflow2.5.1": {
321+
"py_versions": ["py37"],
322+
"registries": {
323+
"af-south-1": "626614931356",
324+
"ap-east-1": "871362719292",
325+
"ap-northeast-1": "763104351884",
326+
"ap-northeast-2": "763104351884",
327+
"ap-northeast-3": "364406365360",
328+
"ap-south-1": "763104351884",
329+
"ap-southeast-1": "763104351884",
330+
"ap-southeast-2": "763104351884",
331+
"ca-central-1": "763104351884",
332+
"cn-north-1": "727897471807",
333+
"cn-northwest-1": "727897471807",
334+
"eu-central-1": "763104351884",
335+
"eu-north-1": "763104351884",
336+
"eu-south-1": "692866216735",
337+
"eu-west-1": "763104351884",
338+
"eu-west-2": "763104351884",
339+
"eu-west-3": "763104351884",
340+
"me-south-1": "217643126080",
341+
"sa-east-1": "763104351884",
342+
"us-east-1": "763104351884",
343+
"us-east-2": "763104351884",
344+
"us-gov-west-1": "442386744353",
345+
"us-iso-east-1": "886529160074",
346+
"us-west-1": "763104351884",
347+
"us-west-2": "763104351884"
348+
},
349+
"repository": "huggingface-tensorflow-training",
350+
"container_version": {"gpu": "cu112-ubuntu18.04"}
351+
}
281352
}
282353
}
283354
},
284355

285356
"inference": {
286357
"processors": ["gpu", "cpu"],
287358
"version_aliases": {
288-
"4.6": "4.6.1"
359+
"4.6": "4.6.1",
360+
"4.10": "4.10.2"
289361
},
290362
"versions": {
291363
"4.6.1": {
@@ -389,6 +461,76 @@
389461
"repository": "huggingface-tensorflow-inference",
390462
"container_version": {"gpu":"cu110-ubuntu18.04", "cpu":"ubuntu18.04" }
391463
}
464+
},
465+
"4.10.2": {
466+
"version_aliases": {
467+
"pytorch1.9": "pytorch1.9.0",
468+
"tensorflow2.5": "tensorflow2.5.1"
469+
},
470+
"pytorch1.9.0": {
471+
"py_versions": ["py38"],
472+
"registries": {
473+
"af-south-1": "626614931356",
474+
"ap-east-1": "871362719292",
475+
"ap-northeast-1": "763104351884",
476+
"ap-northeast-2": "763104351884",
477+
"ap-northeast-3": "364406365360",
478+
"ap-south-1": "763104351884",
479+
"ap-southeast-1": "763104351884",
480+
"ap-southeast-2": "763104351884",
481+
"ca-central-1": "763104351884",
482+
"cn-north-1": "727897471807",
483+
"cn-northwest-1": "727897471807",
484+
"eu-central-1": "763104351884",
485+
"eu-north-1": "763104351884",
486+
"eu-west-1": "763104351884",
487+
"eu-west-2": "763104351884",
488+
"eu-west-3": "763104351884",
489+
"eu-south-1": "692866216735",
490+
"me-south-1": "217643126080",
491+
"sa-east-1": "763104351884",
492+
"us-east-1": "763104351884",
493+
"us-east-2": "763104351884",
494+
"us-gov-west-1": "442386744353",
495+
"us-iso-east-1": "886529160074",
496+
"us-west-1": "763104351884",
497+
"us-west-2": "763104351884"
498+
},
499+
"repository": "huggingface-pytorch-inference",
500+
"container_version": {"gpu": "cu111-ubuntu20.04", "cpu": "ubuntu20.04" }
501+
},
502+
"tensorflow2.5.1": {
503+
"py_versions": ["py37"],
504+
"registries": {
505+
"af-south-1": "626614931356",
506+
"ap-east-1": "871362719292",
507+
"ap-northeast-1": "763104351884",
508+
"ap-northeast-2": "763104351884",
509+
"ap-northeast-3": "364406365360",
510+
"ap-south-1": "763104351884",
511+
"ap-southeast-1": "763104351884",
512+
"ap-southeast-2": "763104351884",
513+
"ca-central-1": "763104351884",
514+
"cn-north-1": "727897471807",
515+
"cn-northwest-1": "727897471807",
516+
"eu-central-1": "763104351884",
517+
"eu-north-1": "763104351884",
518+
"eu-south-1": "692866216735",
519+
"eu-west-1": "763104351884",
520+
"eu-west-2": "763104351884",
521+
"eu-west-3": "763104351884",
522+
"me-south-1": "217643126080",
523+
"sa-east-1": "763104351884",
524+
"us-east-1": "763104351884",
525+
"us-east-2": "763104351884",
526+
"us-gov-west-1": "442386744353",
527+
"us-iso-east-1": "886529160074",
528+
"us-west-1": "763104351884",
529+
"us-west-2": "763104351884"
530+
},
531+
"repository": "huggingface-tensorflow-inference",
532+
"container_version": {"gpu": "cu112-ubuntu18.04", "cpu": "ubuntu18.04" }
533+
}
392534
}
393535
}
394536
}

‎src/sagemaker/model_monitor/clarify_model_monitoring.py

+11-4
Original file line numberDiff line numberDiff line change
@@ -476,9 +476,15 @@ def suggest_baseline(
476476
features_attribute=data_config.features,
477477
)
478478
if model_predicted_label_config is not None:
479-
latest_baselining_job_config.inference_attribute = model_predicted_label_config.label
479+
latest_baselining_job_config.inference_attribute = (
480+
model_predicted_label_config.label
481+
if model_predicted_label_config.label is None
482+
else str(model_predicted_label_config.label)
483+
)
480484
latest_baselining_job_config.probability_attribute = (
481485
model_predicted_label_config.probability
486+
if model_predicted_label_config.probability is None
487+
else str(model_predicted_label_config.probability)
482488
)
483489
latest_baselining_job_config.probability_threshold_attribute = (
484490
model_predicted_label_config.probability_threshold
@@ -827,8 +833,9 @@ def suggest_baseline(
827833
specific explainability method. Currently, only SHAP is supported.
828834
model_config (:class:`~sagemaker.clarify.ModelConfig`): Config of the model and its
829835
endpoint to be created.
830-
model_scores: Index or JSONPath location in the model output for the predicted scores
831-
to be explained. This is not required if the model output is a single score.
836+
model_scores (int or str): Index or JSONPath location in the model output for the
837+
predicted scores to be explained. This is not required if the model output is
838+
a single score.
832839
wait (bool): Whether the call should wait until the job completes (default: False).
833840
logs (bool): Whether to show the logs produced by the job.
834841
Only meaningful when wait is True (default: False).
@@ -865,7 +872,7 @@ def suggest_baseline(
865872
headers=headers,
866873
),
867874
features_attribute=data_config.features,
868-
inference_attribute=model_scores,
875+
inference_attribute=model_scores if model_scores is None else str(model_scores),
869876
)
870877
self.latest_baselining_job_name = baselining_job_name
871878
self.latest_baselining_job = ClarifyBaseliningJob(

‎src/sagemaker/session.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -3554,9 +3554,12 @@ def get_caller_identity_arn(self):
35543554
user_profile_desc = self.sagemaker_client.describe_user_profile(
35553555
DomainId=domain_id, UserProfileName=user_profile_name
35563556
)
3557-
if user_profile_desc.get("UserSettings") is not None:
3557+
3558+
# First, try to find role in userSettings
3559+
if user_profile_desc.get("UserSettings", {}).get("ExecutionRole"):
35583560
return user_profile_desc["UserSettings"]["ExecutionRole"]
35593561

3562+
# If not found, fallback to the domain
35603563
domain_desc = self.sagemaker_client.describe_domain(DomainId=domain_id)
35613564
return domain_desc["DefaultUserSettings"]["ExecutionRole"]
35623565
except ClientError:

‎src/sagemaker/workflow/properties.py

+34
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ def __init__(
6767
for key, info in members.items():
6868
if Properties._shapes.get(info["shape"], {}).get("type") == "list":
6969
self.__dict__[key] = PropertiesList(f"{path}.{key}", info["shape"])
70+
elif Properties._shapes.get(info["shape"], {}).get("type") == "map":
71+
self.__dict__[key] = PropertiesMap(f"{path}.{key}", info["shape"])
7072
else:
7173
self.__dict__[key] = Properties(f"{path}.{key}", info["shape"])
7274

@@ -109,6 +111,38 @@ def __getitem__(self, item: Union[int, str]):
109111
return self._items.get(item)
110112

111113

114+
class PropertiesMap(Properties):
115+
"""PropertiesMap for use in workflow expressions."""
116+
117+
def __init__(self, path: str, shape_name: str = None):
118+
"""Create a PropertiesMap instance representing the given shape.
119+
120+
Args:
121+
path (str): The parent path of the PropertiesMap instance.
122+
shape_name (str): The botocore sagemaker service model shape name.
123+
"""
124+
super(PropertiesMap, self).__init__(path, shape_name)
125+
self.shape_name = shape_name
126+
self._items: Dict[Union[int, str], Properties] = dict()
127+
128+
def __getitem__(self, item: Union[int, str]):
129+
"""Populate the indexing item with a Property, for both lists and dictionaries.
130+
131+
Args:
132+
item (Union[int, str]): The index of the item in sequence.
133+
"""
134+
if item not in self._items.keys():
135+
shape = Properties._shapes.get(self.shape_name)
136+
member = shape["value"]["shape"]
137+
if isinstance(item, str):
138+
property_item = Properties(f"{self._path}['{item}']", member)
139+
else:
140+
property_item = Properties(f"{self._path}[{item}]", member)
141+
self._items[item] = property_item
142+
143+
return self._items.get(item)
144+
145+
112146
@attr.s
113147
class PropertyFile(Expression):
114148
"""Provides a property file struct.

0 commit comments

Comments
 (0)