Merge branch 'master' into script-path

Author: laurenyu

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v1.50.8 (2020-01-30)
+
+### Bug Fixes and Other Changes
+
+ * disable Debugger defaults in unsupported regions
+ * modify session and kms_utils to check for S3 bucket before creation
+ * update docker-compose and PyYAML dependencies
+ * enable smdebug for Horovod (MPI) training setup
+ * create lib dir for dependencies safely (only if it doesn't exist yet).
+ * create the correct session for MultiDataModel
+
+### Documentation Changes
+
+ * update links to the local mode notebooks examples.
+ * Remove outdated badges from README
+ * update links to TF notebook examples to link to script mode examples.
+ * clean up headings, verb tenses, names, etc. in MXNet overview
+ * Update SageMaker operator Helm chart installation guide
+
+### Testing and Release Infrastructure
+
+ * choose faster notebook for notebook PR build
+ * properly fail PR build if has-matching-changes fails
+ * properly fail PR build if has-matching-changes fails
+
 ## v1.50.7 (2020-01-20)
 
 ### Bug fixes and other changes

README.rst

@@ -171,7 +171,7 @@ MXNet SageMaker Estimators
 
 By using MXNet SageMaker Estimators, you can train and host MXNet models on Amazon SageMaker.
 
-Supported versions of MXNet: ``0.12.1``, ``1.0.0``, ``1.1.0``, ``1.2.1``, ``1.3.0``, ``1.4.0``, ``1.4.1``.
+Supported versions of MXNet: ``0.12.1``, ``1.0.0``, ``1.1.0``, ``1.2.1``, ``1.3.0``, ``1.4.0``, ``1.4.1``, ``1.6.0``.
 
 Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``, ``1.4.1``.
 
@@ -187,9 +187,9 @@ TensorFlow SageMaker Estimators
 
 By using TensorFlow SageMaker Estimators, you can train and host TensorFlow models on Amazon SageMaker.
 
-Supported versions of TensorFlow: ``1.4.1``, ``1.5.0``, ``1.6.0``, ``1.7.0``, ``1.8.0``, ``1.9.0``, ``1.10.0``, ``1.11.0``, ``1.12.0``, ``1.13.1``, ``1.14.``, ``1.15.0``, ``2.0.0``.
+Supported versions of TensorFlow: ``1.4.1``, ``1.5.0``, ``1.6.0``, ``1.7.0``, ``1.8.0``, ``1.9.0``, ``1.10.0``, ``1.11.0``, ``1.12.0``, ``1.13.1``, ``1.14.0``, ``1.15.0``, ``2.0.0``.
 
-Supported versions of TensorFlow for Elastic Inference: ``1.11.0``, ``1.12.0``, ``1.13.1``, ``1.14``.
+Supported versions of TensorFlow for Elastic Inference: ``1.11.0``, ``1.12.0``, ``1.13.1``, ``1.14.0``.
 
 We recommend that you use the latest supported version, because that's where we focus most of our development efforts.
 

VERSION

@@ -1 +1 @@
-1.50.8.dev0
+1.50.9.dev0

doc/overview.rst

@@ -818,8 +818,10 @@ Here is an end-to-end example:
 
 For detailed examples of running Docker in local mode, see:
 
-- `TensorFlow local mode example notebook <https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker-python-sdk/tensorflow_distributed_mnist/tensorflow_local_mode_mnist.ipynb>`__.
-- `MXNet local mode example notebook <https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker-python-sdk/mxnet_gluon_mnist/mnist_with_gluon_local_mode.ipynb>`__.
+- `TensorFlow local mode example notebook <https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker-python-sdk/tensorflow_script_mode_using_shell_commands/tensorflow_script_mode_using_shell_commands.ipynb>`__.
+- `MXNet local mode CPU example notebook <https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker-python-sdk/mxnet_gluon_mnist/mxnet_mnist_with_gluon_local_mode.ipynb>`__.
+- `MXNet local mode GPU example notebook <https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker-python-sdk/mxnet_gluon_cifar10/mxnet_cifar10_local_mode.ipynb>`__.
+- `PyTorch local mode example notebook <https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker-python-sdk/pytorch_cnn_cifar10/pytorch_local_mode_cifar10.ipynb>`__.
 
 You can also find these notebooks in the **SageMaker Python SDK** section of the **SageMaker Examples** section in a notebook instance.
 For information about using sample notebooks in a SageMaker notebook instance, see `Use Example Notebooks <https://docs.aws.amazon.com/sagemaker/latest/dg/howitworks-nbexamples.html>`__

setup.py

@@ -50,8 +50,8 @@ def read_version():
     "analytics": ["pandas"],
     "local": [
         "urllib3>=1.21.1,<1.26,!=1.25.0,!=1.25.1",
-        "docker-compose>=1.23.0",
-        "PyYAML>=3.10, <5",  # PyYAML version has to match docker-compose requirements
+        "docker-compose>=1.25.2",
+        "PyYAML>=5.3, <6",  # PyYAML version has to match docker-compose requirements
     ],
     "tensorflow": ["tensorflow>=1.3.0"],
 }

src/sagemaker/estimator.py

@@ -38,6 +38,7 @@
     parse_s3_url,
     UploadedCode,
     validate_source_dir,
+    _region_supports_debugger,
 )
 from sagemaker.job import _Job
 from sagemaker.local import LocalSession
@@ -1674,7 +1675,9 @@ def _validate_and_set_debugger_configs(self):
         """
         Set defaults for debugging
         """
-        if self.debugger_hook_config is None:
+        if self.debugger_hook_config is None and _region_supports_debugger(
+            self.sagemaker_session.boto_region_name
+        ):
             self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
         elif not self.debugger_hook_config:
             self.debugger_hook_config = None

src/sagemaker/fw_utils.py

@@ -84,6 +84,8 @@
     "pytorch-serving": [1, 2, 0],
 }
 
+DEBUGGER_UNSUPPORTED_REGIONS = ["us-gov-west-1", "us-iso-east-1"]
+
 
 def is_version_equal_or_higher(lowest_version, framework_version):
     """Determine whether the ``framework_version`` is equal to or higher than
@@ -504,3 +506,16 @@ def python_deprecation_warning(framework, latest_supported_version):
     return PYTHON_2_DEPRECATION_WARNING.format(
         framework=framework, latest_supported_version=latest_supported_version
     )
+
+
+def _region_supports_debugger(region_name):
+    """Returns boolean indicating whether the region supports Amazon SageMaker Debugger.
+
+    Args:
+        region_name (str): Name of the region to check against.
+
+    Returns:
+        bool: Whether or not the region supports Amazon SageMaker Debugger.
+
+    """
+    return region_name.lower() not in DEBUGGER_UNSUPPORTED_REGIONS

src/sagemaker/mxnet/README.rst

@@ -6,7 +6,7 @@ With the SageMaker Python SDK, you can train and host MXNet models on Amazon Sag
 
 Supported versions of MXNet: ``0.12.1``, ``1.0.0``, ``1.1.0``, ``1.2.1``, ``1.3.0``, ``1.4.0``, ``1.4.1``, ``1.6.0``.
 
-Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``, ``1.4.1``, ``1.6.0``.
+Supported versions of MXNet for Elastic Inference: ``1.3.0``, ``1.4.0``, ``1.4.1``.
 
 For information about using MXNet with the SageMaker Python SDK, see https://sagemaker.readthedocs.io/en/stable/using_mxnet.html.
 

src/sagemaker/mxnet/model.py

@@ -144,7 +144,9 @@ def prepare_container_def(self, instance_type, accelerator_type=None):
         deploy_image = self.image
         if not deploy_image:
             region_name = self.sagemaker_session.boto_session.region_name
-            deploy_image = self.serving_image_uri(region_name, instance_type)
+            deploy_image = self.serving_image_uri(
+                region_name, instance_type, accelerator_type=accelerator_type
+            )
 
         deploy_key_prefix = model_code_key_prefix(self.key_prefix, self.name, deploy_image)
         self._upload_code(deploy_key_prefix, self._is_mms_version())
@@ -157,24 +159,32 @@ def prepare_container_def(self, instance_type, accelerator_type=None):
             deploy_image, self.repacked_model_data or self.model_data, deploy_env
         )
 
-    def serving_image_uri(self, region_name, instance_type):
+    def serving_image_uri(self, region_name, instance_type, accelerator_type=None):
         """Create a URI for the serving image.
 
         Args:
             region_name (str): AWS region where the image is uploaded.
             instance_type (str): SageMaker instance type. Used to determine device type
                 (cpu/gpu/family-specific optimized).
+            accelerator_type (str): The Elastic Inference accelerator type to
+                deploy to the instance for loading and making inferences to the
+                model (default: None). For example, 'ml.eia1.medium'.
 
         Returns:
             str: The appropriate image URI based on the given parameters.
 
         """
         framework_name = self.__framework_name__
         if self._is_mms_version():
-            framework_name += "-serving"
+            framework_name = "{}-serving".format(framework_name)
 
         return create_image_uri(
-            region_name, framework_name, instance_type, self.framework_version, self.py_version
+            region_name,
+            framework_name,
+            instance_type,
+            self.framework_version,
+            self.py_version,
+            accelerator_type=accelerator_type,
         )
 
     def _is_mms_version(self):
@@ -184,6 +194,6 @@ def _is_mms_version(self):
         Returns:
             bool: If the framework version corresponds to an image using MMS.
         """
-        return packaging.version.Version(self.framework_version) >= packaging.version.Version(
-            self._LOWEST_MMS_VERSION
-        )
+        lowest_mms_version = packaging.version.Version(self._LOWEST_MMS_VERSION)
+        framework_version = packaging.version.Version(self.framework_version)
+        return framework_version >= lowest_mms_version

src/sagemaker/pytorch/model.py

@@ -15,7 +15,6 @@
 
 import logging
 import packaging.version
-from sagemaker import fw_utils
 
 import sagemaker
 from sagemaker.fw_utils import (
@@ -137,34 +136,21 @@ def prepare_container_def(self, instance_type, accelerator_type=None):
                 For example, 'ml.p2.xlarge'.
             accelerator_type (str): The Elastic Inference accelerator type to
                 deploy to the instance for loading and making inferences to the
-                model. For example, 'ml.eia1.medium'.
+                model. Currently unsupported with PyTorch.
 
         Returns:
             dict[str, str]: A container definition object usable with the
             CreateModel API.
         """
-        lowest_mms_version = packaging.version.Version(self._LOWEST_MMS_VERSION)
-        framework_version = packaging.version.Version(self.framework_version)
-        is_mms_version = framework_version >= lowest_mms_version
-
         deploy_image = self.image
         if not deploy_image:
             region_name = self.sagemaker_session.boto_session.region_name
-
-            framework_name = self.__framework_name__
-            if is_mms_version:
-                framework_name += "-serving"
-
-            deploy_image = create_image_uri(
-                region_name,
-                framework_name,
-                instance_type,
-                self.framework_version,
-                self.py_version,
-                accelerator_type=accelerator_type,
+            deploy_image = self.serving_image_uri(
+                region_name, instance_type, accelerator_type=accelerator_type
             )
+
         deploy_key_prefix = model_code_key_prefix(self.key_prefix, self.name, deploy_image)
-        self._upload_code(deploy_key_prefix, repack=is_mms_version)
+        self._upload_code(deploy_key_prefix, repack=self._is_mms_version())
         deploy_env = dict(self.env)
         deploy_env.update(self._framework_env_vars())
 
@@ -174,22 +160,41 @@ def prepare_container_def(self, instance_type, accelerator_type=None):
             deploy_image, self.repacked_model_data or self.model_data, deploy_env
         )
 
-    def serving_image_uri(self, region_name, instance_type):
+    def serving_image_uri(self, region_name, instance_type, accelerator_type=None):
         """Create a URI for the serving image.
 
         Args:
             region_name (str): AWS region where the image is uploaded.
             instance_type (str): SageMaker instance type. Used to determine device type
                 (cpu/gpu/family-specific optimized).
+            accelerator_type (str): The Elastic Inference accelerator type to
+                deploy to the instance for loading and making inferences to the
+                model. Currently unsupported with PyTorch.
 
         Returns:
             str: The appropriate image URI based on the given parameters.
 
         """
-        return fw_utils.create_image_uri(
+        framework_name = self.__framework_name__
+        if self._is_mms_version():
+            framework_name = "{}-serving".format(framework_name)
+
+        return create_image_uri(
             region_name,
-            "-".join([self.__framework_name__, "serving"]),
+            framework_name,
             instance_type,
             self.framework_version,
             self.py_version,
+            accelerator_type=accelerator_type,
         )
+
+    def _is_mms_version(self):
+        """Whether the framework version corresponds to an inference image using
+        the Multi-Model Server (https://github.com/awslabs/multi-model-server).
+
+        Returns:
+            bool: If the framework version corresponds to an image using MMS.
+        """
+        lowest_mms_version = packaging.version.Version(self._LOWEST_MMS_VERSION)
+        framework_version = packaging.version.Version(self.framework_version)
+        return framework_version >= lowest_mms_version

src/sagemaker/session.py

@@ -342,40 +342,58 @@ def default_bucket(self):
             ).get_caller_identity()["Account"]
             default_bucket = "sagemaker-{}-{}".format(region, account)
 
-        s3 = self.boto_session.resource("s3")
-        try:
-            # 'us-east-1' cannot be specified because it is the default region:
-            # https://github.com/boto/boto3/issues/125
-            if region == "us-east-1":
-                s3.create_bucket(Bucket=default_bucket)
-            else:
-                s3.create_bucket(
-                    Bucket=default_bucket, CreateBucketConfiguration={"LocationConstraint": region}
-                )
-
-            LOGGER.info("Created S3 bucket: %s", default_bucket)
-        except ClientError as e:
-            error_code = e.response["Error"]["Code"]
-            message = e.response["Error"]["Message"]
-
-            if error_code == "BucketAlreadyOwnedByYou":
-                pass
-            elif (
-                error_code == "OperationAborted" and "conflicting conditional operation" in message
-            ):
-                # If this bucket is already being concurrently created, we don't need to create it
-                # again.
-                pass
-            elif error_code == "TooManyBuckets":
-                # Succeed if the default bucket exists
-                s3.meta.client.head_bucket(Bucket=default_bucket)
-            else:
-                raise
+        self._create_s3_bucket_if_it_does_not_exist(bucket_name=default_bucket, region=region)
 
         self._default_bucket = default_bucket
 
         return self._default_bucket
 
+    def _create_s3_bucket_if_it_does_not_exist(self, bucket_name, region):
+        """Creates an S3 Bucket if it does not exist.
+        Also swallows a few common exceptions that indicate that the bucket already exists or
+        that it is being created.
+
+        Args:
+            bucket_name (str): Name of the S3 bucket to be created.
+            region (str): The region in which to create the bucket.
+
+        Raises:
+            botocore.exceptions.ClientError: If S3 throws an unexpected exception during bucket
+                creation.
+                If the exception is due to the bucket already existing or
+                already being created, no exception is raised.
+
+        """
+        bucket = self.boto_session.resource("s3", region_name=region).Bucket(name=bucket_name)
+        if bucket.creation_date is None:
+            try:
+                s3 = self.boto_session.resource("s3", region_name=region)
+                if region == "us-east-1":
+                    # 'us-east-1' cannot be specified because it is the default region:
+                    # https://github.com/boto/boto3/issues/125
+                    s3.create_bucket(Bucket=bucket_name)
+                else:
+                    s3.create_bucket(
+                        Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": region}
+                    )
+
+                LOGGER.info("Created S3 bucket: %s", bucket_name)
+            except ClientError as e:
+                error_code = e.response["Error"]["Code"]
+                message = e.response["Error"]["Message"]
+
+                if error_code == "BucketAlreadyOwnedByYou":
+                    pass
+                elif (
+                    error_code == "OperationAborted"
+                    and "conflicting conditional operation" in message
+                ):
+                    # If this bucket is already being concurrently created, we don't need to create
+                    # it again.
+                    pass
+                else:
+                    raise
+
     def train(  # noqa: C901
         self,
         input_mode,

src/sagemaker/tensorflow/estimator.py

@@ -723,7 +723,9 @@ def _validate_and_set_debugger_configs(self):
                 )
             self.debugger_hook_config = None
             self.debugger_rule_configs = None
-        elif self.debugger_hook_config is None:
+        elif self.debugger_hook_config is None and fw._region_supports_debugger(
+            self.sagemaker_session.boto_session.region_name
+        ):
             # Set defaults for debugging.
             self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)