Skip to content

Commit 193c23e

Browse files
ranzviranzvi
authored
Merge branch 'master' into fix-localmode-subprocess-termination
2 parents 5e33b4f + 7b07b90 commit 193c23e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+764
-86
lines changed

‎CHANGELOG.md

+72
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,77 @@
11
# Changelog
22

3+
## v2.58.0 (2021-08-31)
4+
5+
### Features
6+
7+
* update debugger for KIX
8+
* support displayName and description for pipeline steps
9+
10+
### Bug Fixes and Other Changes
11+
12+
* localmode subprocess parent process not sending SIGTERM to child
13+
14+
## v2.57.0 (2021-08-30)
15+
16+
### Deprecations and Removals
17+
18+
* Remove stale S3DownloadMode from test_session.py
19+
20+
### Features
21+
22+
* update clarify imageURI for KIX
23+
24+
### Bug Fixes and Other Changes
25+
26+
* propagate KMS key to model.deploy
27+
* Propagate tags and VPC configs to repack model steps
28+
29+
## v2.56.0 (2021-08-26)
30+
31+
### Features
32+
33+
* Add NEO KIX Configuration
34+
* Algorithms region launch on KIX
35+
36+
### Bug Fixes and Other Changes
37+
38+
* remove dots from CHANGELOG
39+
40+
## v2.55.0 (2021-08-25)
41+
42+
### Features
43+
44+
* Add information of Amazon-provided analysis image used by Model Monitor
45+
46+
### Bug Fixes and Other Changes
47+
48+
* Update Changelog to fix release
49+
* Fixing the order of populating container list
50+
* pass network isolation config to pipelineModel
51+
* Deference symbolic link when create tar file
52+
* multiprocess issue in feature_group.py
53+
* deprecate tag logic on Association
54+
55+
### Documentation Changes
56+
57+
* add dataset_definition to processing page
58+
59+
## v2.54.0 (2021-08-16)
60+
61+
### Features
62+
63+
* add pytorch 1.5.1 eia configuration
64+
65+
### Bug Fixes and Other Changes
66+
67+
* issue #2253 where Processing job in Local mode would call Describe API
68+
69+
## v2.53.0 (2021-08-12)
70+
71+
### Features
72+
73+
* support tuning step parameter range parameterization + support retry strategy in tuner
74+
375
## v2.52.2.post0 (2021-08-11)
476

577
### Documentation Changes

‎VERSION

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.52.3.dev0
1+
2.58.1.dev0

‎doc/api/utility/inputs.rst

+5-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,8 @@ Inputs
55
:members:
66
:undoc-members:
77
:show-inheritance:
8-
:noindex:
8+
9+
.. automodule:: sagemaker.dataset_definition.inputs
10+
:members:
11+
:undoc-members:
12+
:show-inheritance:

‎setup.py

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ def read_version():
5252
"urllib3>=1.21.1,!=1.25,!=1.25.1",
5353
"docker-compose>=1.25.2",
5454
"PyYAML>=5.3, <6", # PyYAML version has to match docker-compose requirements
55+
"psutil",
5556
],
5657
"scipy": ["scipy>=0.19.0"],
5758
}

‎src/sagemaker/automl/automl.py

+2
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,7 @@ def create_model(
328328
predictor_cls=predictor_cls,
329329
name=name,
330330
vpc_config=vpc_config,
331+
enable_network_isolation=enable_network_isolation,
331332
sagemaker_session=sagemaker_session or self.sagemaker_session,
332333
)
333334
return pipeline
@@ -419,6 +420,7 @@ def deploy(
419420
serializer=serializer,
420421
deserializer=deserializer,
421422
endpoint_name=endpoint_name,
423+
kms_key=model_kms_key,
422424
tags=tags,
423425
wait=wait,
424426
)

‎src/sagemaker/dataset_definition/inputs.py

+6-7
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ class RedshiftDatasetDefinition(ApiObject):
2727
2828
With this input, SQL queries will be executed using Redshift to generate datasets to S3.
2929
30-
Attributes:
30+
Parameters:
3131
cluster_id (str): The Redshift cluster Identifier.
3232
database (str): The name of the Redshift database used in Redshift query execution.
3333
db_user (str): The database user name used in Redshift query execution.
@@ -60,7 +60,7 @@ class AthenaDatasetDefinition(ApiObject):
6060
6161
With this input, SQL queries will be executed using Athena to generate datasets to S3.
6262
63-
Attributes:
63+
Parameters:
6464
catalog (str): The name of the data catalog used in Athena query execution.
6565
database (str): The name of the database used in the Athena query execution.
6666
query_string (str): The SQL query statements, to be executed.
@@ -87,7 +87,7 @@ class AthenaDatasetDefinition(ApiObject):
8787
class DatasetDefinition(ApiObject):
8888
"""DatasetDefinition input.
8989
90-
Attributes:
90+
Parameters:
9191
data_distribution_type (str): Whether the generated dataset is FullyReplicated or
9292
ShardedByS3Key (default).
9393
input_mode (str): Whether to use File or Pipe input mode. In File (default) mode, Amazon
@@ -98,9 +98,8 @@ class DatasetDefinition(ApiObject):
9898
local_path (str): The local path where you want Amazon SageMaker to download the Dataset
9999
Definition inputs to run a processing job. LocalPath is an absolute path to the input
100100
data. This is a required parameter when `AppManaged` is False (default).
101-
redshift_dataset_definition
102-
(:class:`~sagemaker.dataset_definition.inputs.RedshiftDatasetDefinition`): Redshift
103-
dataset definition.
101+
redshift_dataset_definition (:class:`~sagemaker.dataset_definition.inputs.RedshiftDatasetDefinition`):
102+
Configuration for Redshift Dataset Definition input.
104103
athena_dataset_definition (:class:`~sagemaker.dataset_definition.inputs.AthenaDatasetDefinition`):
105104
Configuration for Athena Dataset Definition input.
106105
"""
@@ -126,7 +125,7 @@ class S3Input(ApiObject):
126125
S3 list operations are not strongly consistent.
127126
Use ManifestFile if strong consistency is required.
128127
129-
Attributes:
128+
Parameters:
130129
s3_uri (str): the path to a specific S3 object or a S3 prefix
131130
local_path (str): the path to a local directory. If not provided, skips data download
132131
by SageMaker platform.

‎src/sagemaker/feature_store/feature_group.py

+32-10
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,8 @@ def _ingest_single_batch(
207207
for row in data_frame[start_index:end_index].itertuples():
208208
record = [
209209
FeatureValue(
210-
feature_name=data_frame.columns[index - 1], value_as_string=str(row[index])
210+
feature_name=data_frame.columns[index - 1],
211+
value_as_string=str(row[index]),
211212
)
212213
for index in range(1, len(row))
213214
if pd.notna(row[index])
@@ -270,13 +271,24 @@ def _run_multi_process(self, data_frame: DataFrame, wait=True, timeout=None):
270271
timeout (Union[int, float]): ``concurrent.futures.TimeoutError`` will be raised
271272
if timeout is reached.
272273
"""
274+
# pylint: disable=I1101
273275
batch_size = math.ceil(data_frame.shape[0] / self.max_processes)
276+
# pylint: enable=I1101
274277

275278
args = []
276279
for i in range(self.max_processes):
277280
start_index = min(i * batch_size, data_frame.shape[0])
278281
end_index = min(i * batch_size + batch_size, data_frame.shape[0])
279-
args += [(data_frame[start_index:end_index], start_index, timeout)]
282+
args += [
283+
(
284+
self.max_workers,
285+
self.feature_group_name,
286+
self.sagemaker_fs_runtime_client_config,
287+
data_frame[start_index:end_index],
288+
start_index,
289+
timeout,
290+
)
291+
]
280292

281293
def init_worker():
282294
# ignore keyboard interrupts in child processes.
@@ -285,13 +297,21 @@ def init_worker():
285297
self._processing_pool = ProcessingPool(self.max_processes, init_worker)
286298
self._processing_pool.restart(force=True)
287299

288-
f = lambda x: self._run_multi_threaded(*x) # noqa: E731
300+
f = lambda x: IngestionManagerPandas._run_multi_threaded(*x) # noqa: E731
289301
self._async_result = self._processing_pool.amap(f, args)
290302

291303
if wait:
292304
self.wait(timeout=timeout)
293305

294-
def _run_multi_threaded(self, data_frame: DataFrame, row_offset=0, timeout=None) -> List[int]:
306+
@staticmethod
307+
def _run_multi_threaded(
308+
max_workers: int,
309+
feature_group_name: str,
310+
sagemaker_fs_runtime_client_config: Config,
311+
data_frame: DataFrame,
312+
row_offset=0,
313+
timeout=None,
314+
) -> List[int]:
295315
"""Start the ingestion process.
296316
297317
Args:
@@ -305,21 +325,23 @@ def _run_multi_threaded(self, data_frame: DataFrame, row_offset=0, timeout=None)
305325
Returns:
306326
List of row indices that failed to be ingested.
307327
"""
308-
executor = ThreadPoolExecutor(max_workers=self.max_workers)
309-
batch_size = math.ceil(data_frame.shape[0] / self.max_workers)
328+
executor = ThreadPoolExecutor(max_workers=max_workers)
329+
# pylint: disable=I1101
330+
batch_size = math.ceil(data_frame.shape[0] / max_workers)
331+
# pylint: enable=I1101
310332

311333
futures = {}
312-
for i in range(self.max_workers):
334+
for i in range(max_workers):
313335
start_index = min(i * batch_size, data_frame.shape[0])
314336
end_index = min(i * batch_size + batch_size, data_frame.shape[0])
315337
futures[
316338
executor.submit(
317-
self._ingest_single_batch,
318-
feature_group_name=self.feature_group_name,
339+
IngestionManagerPandas._ingest_single_batch,
340+
feature_group_name=feature_group_name,
319341
data_frame=data_frame,
320342
start_index=start_index,
321343
end_index=end_index,
322-
client_config=self.sagemaker_fs_runtime_client_config,
344+
client_config=sagemaker_fs_runtime_client_config,
323345
)
324346
] = (start_index + row_offset, end_index + row_offset)
325347

‎src/sagemaker/image_uri_config/blazingtext.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "286214385809",
88
"ap-northeast-1": "501404015308",
99
"ap-northeast-2": "306986355934",
10+
"ap-northeast-3": "867004704886",
1011
"ap-south-1": "991648021394",
1112
"ap-southeast-1": "475088953585",
1213
"ap-southeast-2": "544295431143",

‎src/sagemaker/image_uri_config/clarify.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "098760798382",
88
"ap-northeast-1": "377024640650",
99
"ap-northeast-2": "263625296855",
10+
"ap-northeast-3": "912233562940",
1011
"ap-south-1": "452307495513",
1112
"ap-southeast-1": "834264404009",
1213
"ap-southeast-2": "007051062584",

‎src/sagemaker/image_uri_config/debugger.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "199566480951",
88
"ap-northeast-1": "430734990657",
99
"ap-northeast-2": "578805364391",
10+
"ap-northeast-3": "479947661362",
1011
"ap-south-1": "904829902805",
1112
"ap-southeast-1": "972752614525",
1213
"ap-southeast-2": "184798709955",

‎src/sagemaker/image_uri_config/factorization-machines.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "286214385809",
88
"ap-northeast-1": "351501993468",
99
"ap-northeast-2": "835164637446",
10+
"ap-northeast-3": "867004704886",
1011
"ap-south-1": "991648021394",
1112
"ap-southeast-1": "475088953585",
1213
"ap-southeast-2": "712309505854",

‎src/sagemaker/image_uri_config/image-classification-neo.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "110948597952",
88
"ap-northeast-1": "941853720454",
99
"ap-northeast-2": "151534178276",
10+
"ap-northeast-3": "925152966179",
1011
"ap-south-1": "763008648453",
1112
"ap-southeast-1": "324986816169",
1213
"ap-southeast-2": "355873309152",

‎src/sagemaker/image_uri_config/image-classification.json

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"ap-south-1": "991648021394",
1111
"ap-southeast-1": "475088953585",
1212
"ap-southeast-2": "544295431143",
13+
"ap-northeast-3": "867004704886",
1314
"ca-central-1": "469771592824",
1415
"cn-north-1": "390948362332",
1516
"cn-northwest-1": "387376663083",

‎src/sagemaker/image_uri_config/inferentia-mxnet.json

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"ap-east-1": "110948597952",
1010
"ap-northeast-1": "941853720454",
1111
"ap-northeast-2": "151534178276",
12+
"ap-northeast-3": "925152966179",
1213
"ap-south-1": "763008648453",
1314
"ap-southeast-1": "324986816169",
1415
"ap-southeast-2": "355873309152",

‎src/sagemaker/image_uri_config/inferentia-pytorch.json

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"ap-east-1": "110948597952",
1010
"ap-northeast-1": "941853720454",
1111
"ap-northeast-2": "151534178276",
12+
"ap-northeast-3": "925152966179",
1213
"ap-south-1": "763008648453",
1314
"ap-southeast-1": "324986816169",
1415
"ap-southeast-2": "355873309152",

‎src/sagemaker/image_uri_config/inferentia-tensorflow.json

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"ap-east-1": "110948597952",
1010
"ap-northeast-1": "941853720454",
1111
"ap-northeast-2": "151534178276",
12+
"ap-northeast-3": "925152966179",
1213
"ap-south-1": "763008648453",
1314
"ap-southeast-1": "324986816169",
1415
"ap-southeast-2": "355873309152",

‎src/sagemaker/image_uri_config/ipinsights.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "286214385809",
88
"ap-northeast-1": "351501993468",
99
"ap-northeast-2": "835164637446",
10+
"ap-northeast-3": "867004704886",
1011
"ap-south-1": "991648021394",
1112
"ap-southeast-1": "475088953585",
1213
"ap-southeast-2": "712309505854",

‎src/sagemaker/image_uri_config/kmeans.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "286214385809",
88
"ap-northeast-1": "351501993468",
99
"ap-northeast-2": "835164637446",
10+
"ap-northeast-3": "867004704886",
1011
"ap-south-1": "991648021394",
1112
"ap-southeast-1": "475088953585",
1213
"ap-southeast-2": "712309505854",

‎src/sagemaker/image_uri_config/knn.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33
"versions": {
44
"1": {
55
"registries": {
6-
"af-south-1": "455444449433",
6+
"af-south-1": "455444449433",
77
"ap-east-1": "286214385809",
88
"ap-northeast-1": "351501993468",
99
"ap-northeast-2": "835164637446",
10+
"ap-northeast-3": "867004704886",
1011
"ap-south-1": "991648021394",
1112
"ap-southeast-1": "475088953585",
1213
"ap-southeast-2": "712309505854",

‎src/sagemaker/image_uri_config/linear-learner.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "286214385809",
88
"ap-northeast-1": "351501993468",
99
"ap-northeast-2": "835164637446",
10+
"ap-northeast-3": "867004704886",
1011
"ap-south-1": "991648021394",
1112
"ap-southeast-1": "475088953585",
1213
"ap-southeast-2": "712309505854",

‎src/sagemaker/image_uri_config/model-monitor.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"ap-east-1": "001633400207",
88
"ap-northeast-1": "574779866223",
99
"ap-northeast-2": "709848358524",
10+
"ap-northeast-3": "990339680094",
1011
"ap-south-1": "126357580389",
1112
"ap-southeast-1": "245545462676",
1213
"ap-southeast-2": "563025443158",

‎src/sagemaker/image_uri_config/neo-mxnet.json

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
"ap-east-1": "110948597952",
2424
"ap-northeast-1": "941853720454",
2525
"ap-northeast-2": "151534178276",
26+
"ap-northeast-3": "925152966179",
2627
"ap-south-1": "763008648453",
2728
"ap-southeast-1": "324986816169",
2829
"ap-southeast-2": "355873309152",

0 commit comments

Comments
 (0)