Skip to content

Commit 795e20d

Browse files
ahsan-z-khanahsan-z-khan
authored
Merge branch 'master' into training-inputs-doc-fix
2 parents 75a650a + 1b5020b commit 795e20d

36 files changed

+952
-73
lines changed

‎.githooks/pre-push

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/sh
2+
# this pre-push hook runs style checks and unit tests in python 3.6, 3.7, and 3.8 using tox.
3+
4+
set -e
5+
6+
TOX_PARALLEL_NO_SPINNER=1,
7+
PY_COLORS=0
8+
start_time=`date +%s`
9+
tox -e flake8,pylint,docstyle,black-check,twine --parallel all
10+
./ci-scripts/displaytime.sh 'flake8,pylint,docstyle,black-check,twine' $start_time
11+
start_time=`date +%s`
12+
tox -e sphinx,doc8 --parallel all
13+
./ci-scripts/displaytime.sh 'sphinx,doc8' $start_time
14+
start_time=`date +%s`
15+
tox -e py36,py37,py38 --parallel all -- tests/unit
16+
./ci-scripts/displaytime.sh 'py36,py37,py38 unit' $start_time

‎CHANGELOG.md

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,46 @@
11
# Changelog
22

3+
## v2.65.0 (2021-10-21)
4+
5+
### Features
6+
7+
* modify RLEstimator to use newly generated Ray image (1.6.0)
8+
* network isolation mode for xgboost
9+
* update clarify imageURI for PDT
10+
11+
### Bug Fixes and Other Changes
12+
13+
* retry downstream_trials test
14+
* Add retries to pipeline execution
15+
16+
## v2.64.0 (2021-10-20)
17+
18+
### Deprecations and Removals
19+
20+
* warn for deprecation - Lambda model-predictor
21+
22+
### Features
23+
24+
* Add support for TF 2.5
25+
* Add a pre-push git hook
26+
27+
### Bug Fixes and Other Changes
28+
29+
* add s3_analysis_config_output_path field in DataConfig constructor
30+
* make marketplace jobnames random
31+
32+
## v2.63.2 (2021-10-18)
33+
34+
### Bug Fixes and Other Changes
35+
36+
* Update timeouts for integ tests from 20 to 40
37+
38+
## v2.63.1 (2021-10-14)
39+
40+
### Bug Fixes and Other Changes
41+
42+
* HF estimator attach modified to work with py38
43+
344
## v2.63.0 (2021-10-13)
445

546
### Features

‎README.rst

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,18 @@ You can also run them in parallel:
154154
tox -- -n auto tests/integ
155155

156156

157+
Git Hooks
158+
~~~~~~~~~
159+
160+
to enable all git hooks in the .githooks directory, run these commands in the repository directory:
161+
162+
::
163+
164+
find .git/hooks -type l -exec rm {} \;
165+
find .githooks -type f -exec ln -sf ../../{} .git/hooks/ \;
166+
167+
To enable an individual git hook, simply move it from the .githooks/ directory to the .git/hooks/ directory.
168+
157169
Building Sphinx docs
158170
~~~~~~~~~~~~~~~~~~~~
159171

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.63.1.dev0
1+
2.65.1.dev0

‎doc/v2.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,12 @@ Please use :func:`sagemaker.s3.parse_s3_url` instead.
265265

266266
The class ``sagemaker.session.ModelContainer`` has been deprecated, as it is not needed for creating inference pipelines.
267267

268+
``sagemaker.workflow.condition_step.JsonGet``
269+
---------------------------------------------
270+
271+
The class ``sagemaker.workflow.condition_step.JsonGet`` has been deprecated.
272+
Please use :class:`sagemaker.workflow.functions.JsonGet` instead.
273+
268274
Parameter and Class Name Changes
269275
================================
270276

‎src/sagemaker/clarify.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def __init__(
3131
self,
3232
s3_data_input_path,
3333
s3_output_path,
34+
s3_analysis_config_output_path=None,
3435
label=None,
3536
headers=None,
3637
features=None,
@@ -43,6 +44,9 @@ def __init__(
4344
Args:
4445
s3_data_input_path (str): Dataset S3 prefix/object URI.
4546
s3_output_path (str): S3 prefix to store the output.
47+
s3_analysis_config_output_path (str): S3 prefix to store the analysis_config output
48+
If this field is None, then the s3_output_path will be used
49+
to store the analysis_config output
4650
label (str): Target attribute of the model required by bias metrics (optional for SHAP)
4751
Specified as column name or index for CSV dataset, or as JSONPath for JSONLines.
4852
headers (list[str]): A list of column names in the input dataset.
@@ -61,6 +65,7 @@ def __init__(
6165
)
6266
self.s3_data_input_path = s3_data_input_path
6367
self.s3_output_path = s3_output_path
68+
self.s3_analysis_config_output_path = s3_analysis_config_output_path
6469
self.s3_data_distribution_type = s3_data_distribution_type
6570
self.s3_compression_type = s3_compression_type
6671
self.label = label
@@ -473,7 +478,7 @@ def _run(
473478
json.dump(analysis_config, f)
474479
s3_analysis_config_file = _upload_analysis_config(
475480
analysis_config_file,
476-
data_config.s3_output_path,
481+
data_config.s3_analysis_config_output_path or data_config.s3_output_path,
477482
self.sagemaker_session,
478483
kms_key,
479484
)

‎src/sagemaker/deprecations.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,46 @@ def renamed_warning(phrase):
5050
_warn(f"{phrase} has been renamed")
5151

5252

53+
def deprecation_warn(name, date, msg=None):
54+
"""Raise a warning for soon to be deprecated feature in sagemaker>=2
55+
56+
Args:
57+
name (str): Name of the feature
58+
date (str): the date when the feature will be deprecated
59+
msg (str): the prefix phrase of the warning message.
60+
"""
61+
_warn(f"{name} will be deprecated on {date}.{msg}")
62+
63+
64+
def deprecation_warning(date, msg=None):
65+
"""Decorator for raising deprecation warning for a feature in sagemaker>=2
66+
67+
Args:
68+
date (str): the date when the feature will be deprecated
69+
msg (str): the prefix phrase of the warning message.
70+
71+
Usage:
72+
@deprecation_warning(msg="message", date="date")
73+
def sample_function():
74+
print("xxxx....")
75+
76+
@deprecation_warning(msg="message", date="date")
77+
class SampleClass():
78+
def __init__(self):
79+
print("xxxx....")
80+
81+
"""
82+
83+
def deprecate(obj):
84+
def wrapper(*args, **kwargs):
85+
deprecation_warn(obj.__name__, date, msg)
86+
return obj(*args, **kwargs)
87+
88+
return wrapper
89+
90+
return deprecate
91+
92+
5393
def renamed_kwargs(old_name, new_name, value, kwargs):
5494
"""Checks if the deprecated argument is in kwargs
5595
@@ -106,6 +146,28 @@ def func(*args, **kwargs): # pylint: disable=W0613
106146
return func
107147

108148

149+
def deprecated(obj):
150+
"""Decorator for raising deprecated warning for a feature in sagemaker>=2
151+
152+
Usage:
153+
@deprecated
154+
def sample_function():
155+
print("xxxx....")
156+
157+
@deprecated
158+
class SampleClass():
159+
def __init__(self):
160+
print("xxxx....")
161+
162+
"""
163+
164+
def wrapper(*args, **kwargs):
165+
removed_warning(obj.__name__)
166+
return obj(*args, **kwargs)
167+
168+
return wrapper
169+
170+
109171
def deprecated_function(func, name):
110172
"""Wrap a function with a deprecation warning.
111173

‎src/sagemaker/fw_utils.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
"local_gpu",
6060
)
6161
SM_DATAPARALLEL_SUPPORTED_FRAMEWORK_VERSIONS = {
62-
"tensorflow": ["2.3", "2.3.1", "2.3.2", "2.4", "2.4.1"],
62+
"tensorflow": ["2.3", "2.3.1", "2.3.2", "2.4", "2.4.1", "2.4.3", "2.5", "2.5.0", "2.5.1"],
6363
"pytorch": ["1.6", "1.6.0", "1.7", "1.7.1", "1.8", "1.8.0", "1.8.1", "1.9", "1.9.0"],
6464
}
6565
SMDISTRIBUTED_SUPPORTED_STRATEGIES = ["dataparallel", "modelparallel"]
@@ -533,7 +533,7 @@ def _validate_smdataparallel_args(
533533
if "py3" not in py_version:
534534
err_msg += (
535535
f"Provided py_version {py_version} is not supported by smdataparallel.\n"
536-
"Please specify py_version=py3"
536+
"Please specify py_version>=py3"
537537
)
538538

539539
if err_msg:

‎src/sagemaker/huggingface/estimator.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na
312312
framework_version = None
313313
else:
314314
framework, pt_or_tf = framework.split("-")
315-
tag_pattern = re.compile("^(.*)-transformers(.*)-(cpu|gpu)-(py2|py3[67]?)$")
315+
tag_pattern = re.compile(r"^(.*)-transformers(.*)-(cpu|gpu)-(py2|py3\d*)$")
316316
tag_match = tag_pattern.match(tag)
317317
pt_or_tf_version = tag_match.group(1)
318318
framework_version = tag_match.group(2)

‎src/sagemaker/image_uri_config/autogluon.json

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"training": {
3+
"processors": ["cpu", "gpu"],
4+
"versions": {
5+
"0.3.1": {
6+
"registries": {
7+
"af-south-1": "626614931356",
8+
"ap-east-1": "871362719292",
9+
"ap-northeast-1": "763104351884",
10+
"ap-northeast-2": "763104351884",
11+
"ap-northeast-3": "364406365360",
12+
"ap-south-1": "763104351884",
13+
"ap-southeast-1": "763104351884",
14+
"ap-southeast-2": "763104351884",
15+
"ca-central-1": "763104351884",
16+
"eu-central-1": "763104351884",
17+
"eu-north-1": "763104351884",
18+
"eu-west-1": "763104351884",
19+
"eu-west-2": "763104351884",
20+
"eu-west-3": "763104351884",
21+
"eu-south-1": "692866216735",
22+
"me-south-1": "217643126080",
23+
"sa-east-1": "763104351884",
24+
"us-east-1": "763104351884",
25+
"us-east-2": "763104351884",
26+
"us-gov-west-1": "442386744353",
27+
"us-iso-east-1": "886529160074",
28+
"us-west-1": "763104351884",
29+
"us-west-2": "763104351884"
30+
},
31+
"repository": "autogluon-training",
32+
"py_versions": ["py37"]
33+
}
34+
}
35+
},
36+
"inference": {
37+
"processors": ["cpu"],
38+
"versions": {
39+
"0.3.1": {
40+
"registries": {
41+
"af-south-1": "626614931356",
42+
"ap-east-1": "871362719292",
43+
"ap-northeast-1": "763104351884",
44+
"ap-northeast-2": "763104351884",
45+
"ap-northeast-3": "364406365360",
46+
"ap-south-1": "763104351884",
47+
"ap-southeast-1": "763104351884",
48+
"ap-southeast-2": "763104351884",
49+
"ca-central-1": "763104351884",
50+
"cn-north-1": "727897471807",
51+
"cn-northwest-1": "727897471807",
52+
"eu-central-1": "763104351884",
53+
"eu-north-1": "763104351884",
54+
"eu-west-1": "763104351884",
55+
"eu-west-2": "763104351884",
56+
"eu-west-3": "763104351884",
57+
"eu-south-1": "692866216735",
58+
"me-south-1": "217643126080",
59+
"sa-east-1": "763104351884",
60+
"us-east-1": "763104351884",
61+
"us-east-2": "763104351884",
62+
"us-gov-west-1": "442386744353",
63+
"us-iso-east-1": "886529160074",
64+
"us-west-1": "763104351884",
65+
"us-west-2": "763104351884"
66+
},
67+
"repository": "autogluon-inference",
68+
"py_versions": ["py37"]
69+
}
70+
}
71+
}
72+
}

‎src/sagemaker/image_uri_config/clarify.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525
"us-east-1": "205585389593",
2626
"us-east-2": "211330385671",
2727
"us-west-1": "740489534195",
28-
"us-west-2": "306415355426"
28+
"us-west-2": "306415355426",
29+
"us-gov-west-1": "598674086554"
2930
},
3031
"repository": "sagemaker-clarify-processing"
3132
}

‎src/sagemaker/image_uri_config/ray-pytorch.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,26 @@
2121
},
2222
"repository": "sagemaker-rl-ray-container",
2323
"tag_prefix": "ray-0.8.5-torch"
24+
},
25+
"1.6.0": {
26+
"py_versions": ["py36"],
27+
"registries": {
28+
"ap-northeast-1": "462105765813",
29+
"ap-northeast-2": "462105765813",
30+
"ap-south-1": "462105765813",
31+
"ap-southeast-1": "462105765813",
32+
"ap-southeast-2": "462105765813",
33+
"ca-central-1": "462105765813",
34+
"eu-central-1": "462105765813",
35+
"eu-west-1": "462105765813",
36+
"eu-west-2": "462105765813",
37+
"us-east-1": "462105765813",
38+
"us-east-2": "462105765813",
39+
"us-west-1": "462105765813",
40+
"us-west-2": "462105765813"
41+
},
42+
"repository": "sagemaker-rl-ray-container",
43+
"tag_prefix": "ray-1.6.0-torch"
2444
}
2545
}
2646
}

‎src/sagemaker/image_uri_config/ray-tensorflow.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,26 @@
165165
},
166166
"repository": "sagemaker-rl-ray-container",
167167
"tag_prefix": "ray-0.8.5-tf"
168+
},
169+
"1.6.0": {
170+
"py_versions": ["py37"],
171+
"registries": {
172+
"ap-northeast-1": "462105765813",
173+
"ap-northeast-2": "462105765813",
174+
"ap-south-1": "462105765813",
175+
"ap-southeast-1": "462105765813",
176+
"ap-southeast-2": "462105765813",
177+
"ca-central-1": "462105765813",
178+
"eu-central-1": "462105765813",
179+
"eu-west-1": "462105765813",
180+
"eu-west-2": "462105765813",
181+
"us-east-1": "462105765813",
182+
"us-east-2": "462105765813",
183+
"us-west-1": "462105765813",
184+
"us-west-2": "462105765813"
185+
},
186+
"repository": "sagemaker-rl-ray-container",
187+
"tag_prefix": "ray-1.6.0-tf"
168188
}
169189
}
170190
}

0 commit comments

Comments
 (0)