Merge branch 'master' into fix-localmode-subprocess-termination

Author: ranzvi

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v2.54.0 (2021-08-16)
+
+### Features
+
+ * add pytorch 1.5.1 eia configuration
+
+### Bug Fixes and Other Changes
+
+ * issue #2253 where Processing job in Local mode would call Describe…
+
+## v2.53.0 (2021-08-12)
+
+### Features
+
+ * support tuning step parameter range parameterization + support retry strategy in tuner
+
 ## v2.52.2.post0 (2021-08-11)
 
 ### Documentation Changes

VERSION

@@ -1 +1 @@
-2.52.3.dev0
+2.54.1.dev0

src/sagemaker/automl/automl.py

@@ -328,6 +328,7 @@ def create_model(
             predictor_cls=predictor_cls,
             name=name,
             vpc_config=vpc_config,
+            enable_network_isolation=enable_network_isolation,
             sagemaker_session=sagemaker_session or self.sagemaker_session,
         )
         return pipeline

src/sagemaker/feature_store/feature_group.py

@@ -207,7 +207,8 @@ def _ingest_single_batch(
         for row in data_frame[start_index:end_index].itertuples():
             record = [
                 FeatureValue(
-                    feature_name=data_frame.columns[index - 1], value_as_string=str(row[index])
+                    feature_name=data_frame.columns[index - 1],
+                    value_as_string=str(row[index]),
                 )
                 for index in range(1, len(row))
                 if pd.notna(row[index])
@@ -270,13 +271,24 @@ def _run_multi_process(self, data_frame: DataFrame, wait=True, timeout=None):
             timeout (Union[int, float]): ``concurrent.futures.TimeoutError`` will be raised
                 if timeout is reached.
         """
+        # pylint: disable=I1101
         batch_size = math.ceil(data_frame.shape[0] / self.max_processes)
+        # pylint: enable=I1101
 
         args = []
         for i in range(self.max_processes):
             start_index = min(i * batch_size, data_frame.shape[0])
             end_index = min(i * batch_size + batch_size, data_frame.shape[0])
-            args += [(data_frame[start_index:end_index], start_index, timeout)]
+            args += [
+                (
+                    self.max_workers,
+                    self.feature_group_name,
+                    self.sagemaker_fs_runtime_client_config,
+                    data_frame[start_index:end_index],
+                    start_index,
+                    timeout,
+                )
+            ]
 
         def init_worker():
             # ignore keyboard interrupts in child processes.
@@ -285,13 +297,21 @@ def init_worker():
         self._processing_pool = ProcessingPool(self.max_processes, init_worker)
         self._processing_pool.restart(force=True)
 
-        f = lambda x: self._run_multi_threaded(*x)  # noqa: E731
+        f = lambda x: IngestionManagerPandas._run_multi_threaded(*x)  # noqa: E731
         self._async_result = self._processing_pool.amap(f, args)
 
         if wait:
             self.wait(timeout=timeout)
 
-    def _run_multi_threaded(self, data_frame: DataFrame, row_offset=0, timeout=None) -> List[int]:
+    @staticmethod
+    def _run_multi_threaded(
+        max_workers: int,
+        feature_group_name: str,
+        sagemaker_fs_runtime_client_config: Config,
+        data_frame: DataFrame,
+        row_offset=0,
+        timeout=None,
+    ) -> List[int]:
         """Start the ingestion process.
 
         Args:
@@ -305,21 +325,23 @@ def _run_multi_threaded(self, data_frame: DataFrame, row_offset=0, timeout=None)
         Returns:
             List of row indices that failed to be ingested.
         """
-        executor = ThreadPoolExecutor(max_workers=self.max_workers)
-        batch_size = math.ceil(data_frame.shape[0] / self.max_workers)
+        executor = ThreadPoolExecutor(max_workers=max_workers)
+        # pylint: disable=I1101
+        batch_size = math.ceil(data_frame.shape[0] / max_workers)
+        # pylint: enable=I1101
 
         futures = {}
-        for i in range(self.max_workers):
+        for i in range(max_workers):
             start_index = min(i * batch_size, data_frame.shape[0])
             end_index = min(i * batch_size + batch_size, data_frame.shape[0])
             futures[
                 executor.submit(
-                    self._ingest_single_batch,
-                    feature_group_name=self.feature_group_name,
+                    IngestionManagerPandas._ingest_single_batch,
+                    feature_group_name=feature_group_name,
                     data_frame=data_frame,
                     start_index=start_index,
                     end_index=end_index,
-                    client_config=self.sagemaker_fs_runtime_client_config,
+                    client_config=sagemaker_fs_runtime_client_config,
                 )
             ] = (start_index + row_offset, end_index + row_offset)
 

src/sagemaker/image_uri_config/model-monitor.json

@@ -7,6 +7,7 @@
                 "ap-east-1": "001633400207",
                 "ap-northeast-1": "574779866223",
                 "ap-northeast-2": "709848358524",
+                "ap-northeast-3": "990339680094",
                 "ap-south-1": "126357580389",
                 "ap-southeast-1": "245545462676",
                 "ap-southeast-2": "563025443158",

src/sagemaker/image_uri_config/pytorch.json

@@ -4,16 +4,15 @@
             "cpu"
         ],
         "version_aliases": {
-            "1.3": "1.3.1"
+            "1.3": "1.3.1",
+            "1.5": "1.5.1"
         },
         "versions": {
             "1.3.1": {
                 "py_versions": [
                     "py3"
                 ],
                 "registries": {
-                    "af-south-1": "626614931356",
-                    "ap-east-1": "871362719292",
                     "ap-northeast-1": "763104351884",
                     "ap-northeast-2": "763104351884",
                     "ap-northeast-3": "364406365360",
@@ -26,16 +25,22 @@
                     "eu-central-1": "763104351884",
                     "eu-north-1": "763104351884",
                     "eu-west-1": "763104351884",
-                    "eu-west-2": "763104351884",
-                    "eu-west-3": "763104351884",
-                    "eu-south-1": "692866216735",
-                    "me-south-1": "217643126080",
-                    "sa-east-1": "763104351884",
                     "us-east-1": "763104351884",
                     "us-east-2": "763104351884",
-                    "us-gov-west-1": "442386744353",
-                    "us-iso-east-1": "886529160074",
-                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "pytorch-inference-eia"
+            },
+            "1.5.1": {
+                "py_versions": [
+                    "py3"
+                ],
+                "registries": {
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
                     "us-west-2": "763104351884"
                 },
                 "repository": "pytorch-inference-eia"

src/sagemaker/lineage/association.py

@@ -15,11 +15,14 @@
 
 from typing import Optional, Iterator
 from datetime import datetime
+import logging
 
 from sagemaker.apiutils import _base_types
 from sagemaker.lineage import _api_types
 from sagemaker.lineage._api_types import AssociationSummary
 
+logger = logging.getLogger(__name__)
+
 
 class Association(_base_types.Record):
     """An Amazon SageMaker artifact, which is part of a SageMaker lineage.
@@ -73,6 +76,10 @@ def set_tag(self, tag=None):
         Returns:
             list({str:str}): a list of key value pairs
         """
+        logger.warning(
+            "set_tag on Association is deprecated. Use set_tag on the source or destination\
+             entity instead."
+        )
         return self._set_tags(resource_arn=self.source_arn, tags=[tag])
 
     def set_tags(self, tags=None):
@@ -84,6 +91,10 @@ def set_tags(self, tags=None):
         Returns:
             list({str:str}): a list of key value pairs
         """
+        logger.warning(
+            "set_tags on Association is deprecated. Use set_tags on the source or destination\
+            entity instead."
+        )
         return self._set_tags(resource_arn=self.source_arn, tags=tags)
 
     @classmethod

src/sagemaker/local/local_session.py

@@ -571,6 +571,21 @@ def logs_for_job(self, job_name, wait=False, poll=5, log_type="All"):
         # on local mode.
         pass  # pylint: disable=unnecessary-pass
 
+    def logs_for_processing_job(self, job_name, wait=False, poll=10):
+        """A no-op method meant to override the sagemaker client.
+
+        Args:
+          job_name:
+          wait:  (Default value = False)
+          poll:  (Default value = 10)
+
+        Returns:
+
+        """
+        # override logs_for_job() as it doesn't need to perform any action
+        # on local mode.
+        pass  # pylint: disable=unnecessary-pass
+
 
 class file_input(object):
     """Amazon SageMaker channel configuration for FILE data sources, used in local mode."""

src/sagemaker/utils.py

@@ -331,7 +331,7 @@ def create_tar_file(source_files, target=None):
     else:
         _, filename = tempfile.mkstemp()
 
-    with tarfile.open(filename, mode="w:gz") as t:
+    with tarfile.open(filename, mode="w:gz", dereference=True) as t:
         for sf in source_files:
             # Add all files from the directory into the root of the directory structure of the tar
             t.add(sf, arcname=os.path.basename(sf))

tests/unit/sagemaker/image_uris/test_dlc_frameworks.py

@@ -42,6 +42,14 @@
     "us-gov-west-1": "246785580436",
     "us-iso-east-1": "744548109606",
 }
+ELASTIC_INFERENCE_REGIONS = [
+    "ap-northeast-1",
+    "ap-northeast-2",
+    "eu-west-1",
+    "us-east-1",
+    "us-east-2",
+    "us-west-2",
+]
 
 
 def _test_image_uris(
@@ -385,8 +393,9 @@ def test_pytorch_eia(pytorch_eia_version, pytorch_eia_py_version):
     )
     assert expected == uri
 
-    for region, account in DLC_ALTERNATE_REGION_ACCOUNTS.items():
+    for region in ELASTIC_INFERENCE_REGIONS:
         uri = image_uris.retrieve(region=region, **base_args)
+        account = DLC_ALTERNATE_REGION_ACCOUNTS.get(region, DLC_ACCOUNT)
 
         expected = expected_uris.framework_uri(
             "pytorch-inference-eia",

tests/unit/sagemaker/image_uris/test_model_monitor.py

@@ -20,6 +20,7 @@
     "ap-east-1": "001633400207",
     "ap-northeast-1": "574779866223",
     "ap-northeast-2": "709848358524",
+    "ap-northeast-3": "990339680094",
     "ap-south-1": "126357580389",
     "ap-southeast-1": "245545462676",
     "ap-southeast-2": "563025443158",

tests/unit/test_local_session.py

@@ -551,6 +551,20 @@ def test_describe_transform_job_does_not_exist(LocalSession, _LocalTransformJob)
         local_sagemaker_client.describe_transform_job("transform-job-does-not-exist")
 
 
+@patch("sagemaker.local.image._SageMakerContainer.process")
+@patch("sagemaker.local.local_session.LocalSession")
+def test_logs_for_job(process, LocalSession):
+    local_job_logs = LocalSession.logs_for_job("my-processing-job")
+    assert local_job_logs is not None
+
+
+@patch("sagemaker.local.image._SageMakerContainer.process")
+@patch("sagemaker.local.local_session.LocalSession")
+def test_logs_for_processing_job(process, LocalSession):
+    local_processing_job_logs = LocalSession.logs_for_processing_job("my-processing-job")
+    assert local_processing_job_logs is not None
+
+
 @patch("sagemaker.local.local_session.LocalSession")
 def test_describe_endpoint_config(LocalSession):
     local_sagemaker_client = sagemaker.local.local_session.LocalSagemakerClient()