Merge branch 'master' of https://github.com/aws/sagemaker-python-sdk

Author: nadiaya

CHANGELOG.md

@@ -1,5 +1,47 @@
 # Changelog
 
+## v1.44.3 (2019-11-26)
+
+### Bug fixes and other changes
+
+ * move sagemaker config loading to LocalSession since it is only used for local code support.
+
+### Documentation changes
+
+ * fix docstring wording.
+
+## v1.44.2 (2019-11-25)
+
+### Bug fixes and other changes
+
+ * add pyyaml dependencies to the required list.
+
+### Documentation changes
+
+ * Correct info on code_location parameter
+
+## v1.44.1 (2019-11-21)
+
+### Bug fixes and other changes
+
+ * Remove local mode dependencies from required.
+
+## v1.44.0 (2019-11-21)
+
+### Features
+
+ * separating sagemaker dependencies into more use case specific installable components.
+
+### Bug fixes and other changes
+
+ * remove docker-compose as a required dependency.
+
+## v1.43.5 (2019-11-18)
+
+### Bug fixes and other changes
+
+ * remove red from possible colors when streaming logs
+
 ## v1.43.4.post1 (2019-10-29)
 
 ### Documentation changes

VERSION

@@ -1 +1 @@
-1.43.5.dev0
+1.44.4.dev0

doc/overview.rst

@@ -661,6 +661,12 @@ For example, the ``dataframe`` method gets a pandas dataframe summarizing the as
     # Look at summary of associated training jobs
     my_dataframe = my_tuner_analytics.dataframe()
 
+You can install all necessary for this feature dependencies using pip:
+
+::
+
+    pip install 'sagemaker[analytics]' --upgrade
+
 For more detailed examples of running hyperparameter tuning jobs, see:
 
 - `Using the TensorFlow estimator with hyperparameter tuning <https://github.com/awslabs/amazon-sagemaker-examples/blob/master/hyperparameter_tuning/tensorflow_mnist/hpo_tensorflow_mnist.ipynb>`__
@@ -718,6 +724,12 @@ The SageMaker Python SDK supports local mode, which allows you to create estimat
 This is a great way to test your deep learning scripts before running them in SageMaker's managed training or hosting environments.
 Local Mode is supported for frameworks images (TensorFlow, MXNet, Chainer, PyTorch, and Scikit-Learn) and images you supply yourself.
 
+You can install all necessary for this feature dependencies using pip:
+
+::
+
+    pip install 'sagemaker[local]' --upgrade
+
 We can take the example in  `Using Estimators <#using-estimators>`__ , and use either ``local`` or ``local_gpu`` as the instance type.
 
 .. code:: python

setup.py

@@ -38,13 +38,42 @@ def read_version():
     "numpy>=1.9.0",
     "protobuf>=3.1",
     "scipy>=0.19.0",
-    "urllib3>=1.21, <1.25",
     "protobuf3-to-dict>=0.1.5",
-    "docker-compose>=1.23.0",
     "requests>=2.20.0, <2.21",
-    "fabric>=2.0",
 ]
 
+# Specific use case dependencies
+extras = {
+    "analytics": ["pandas"],
+    "local": [
+        "urllib3>=1.21, <1.25",
+        "docker-compose>=1.23.0",
+        "PyYAML>=3.10, <5",  # PyYAML version has to match docker-compose requirements
+    ],
+    "tensorflow": ["tensorflow>=1.3.0"],
+}
+# Meta dependency groups
+extras["all"] = [item for group in extras.values() for item in group]
+# Tests specific dependencies (do not need to be included in 'all')
+extras["test"] = (
+    [
+        extras["all"],
+        "tox==3.13.1",
+        "flake8",
+        "pytest==4.4.1",
+        "pytest-cov",
+        "pytest-rerunfailures",
+        "pytest-xdist",
+        "mock",
+        "contextlib2",
+        "awslogs",
+        "black==19.3b0 ; python_version >= '3.6'",
+        "stopit==1.1.2",
+        "apache-airflow==1.10.5",
+        "fabric>=2.0",
+    ],
+)
+
 # enum is introduced in Python 3.4. Installing enum back port
 if sys.version_info < (3, 4):
     required_packages.append("enum34>=1.1.6")
@@ -71,23 +100,6 @@ def read_version():
         "Programming Language :: Python :: 3.6",
     ],
     install_requires=required_packages,
-    extras_require={
-        "test": [
-            "tox==3.13.1",
-            "flake8",
-            "pytest==4.4.1",
-            "pytest-cov",
-            "pytest-rerunfailures",
-            "pytest-xdist",
-            "mock",
-            "tensorflow>=1.3.0",
-            "contextlib2",
-            "awslogs",
-            "pandas",
-            "black==19.3b0 ; python_version >= '3.6'",
-            "stopit==1.1.2",
-            "apache-airflow==1.10.5",
-        ]
-    },
+    extras_require=extras,
     entry_points={"console_scripts": ["sagemaker=sagemaker.cli.main:main"]},
 )

src/sagemaker/estimator.py

@@ -1431,11 +1431,10 @@ def __init__(
                 (default: logging.INFO). Valid values are defined in the Python
                 logging module.
             code_location (str): The S3 prefix URI where custom code will be
-                uploaded (default: None). The code file uploaded in S3 is
-                'code_location/source/sourcedir.tar.gz'. If not specified, the
-                default code location is s3://default_bucket/job-name/. And code
-                file uploaded to S3 is
-                s3://default_bucket/job-name/source/sourcedir.tar.gz
+                uploaded (default: None) - don't include a trailing slash since
+                a string prepended with a "/" is appended to ``code_location``. The code
+                file uploaded to S3 is 'code_location/job-name/source/sourcedir.tar.gz'.
+                If not specified, the default ``code location`` is s3://default_bucket/job-name/.
             image_name (str): An alternate image name to use instead of the
                 official Sagemaker image for the framework. This is useful to
                 run one of the Sagemaker supported frameworks with an image

src/sagemaker/local/entities.py

@@ -19,12 +19,19 @@
 import os
 import tempfile
 import time
-import urllib3
 
 import sagemaker.local.data
 from sagemaker.local.image import _SageMakerContainer
 from sagemaker.local.utils import copy_directory_structure, move_to_destination
-from sagemaker.utils import get_config_value
+from sagemaker.utils import DeferredError, get_config_value
+
+try:
+    import urllib3
+except ImportError as e:
+    logging.warning("urllib3 failed to import. Local mode features will be impaired or broken.")
+    # Any subsequent attempt to use urllib3 will raise the ImportError
+    urllib3 = DeferredError(e)
+
 
 logger = logging.getLogger(__name__)
 

src/sagemaker/local/image.py

@@ -29,11 +29,10 @@
 import tarfile
 import tempfile
 
+from distutils.spawn import find_executable
 from threading import Thread
 from six.moves.urllib.parse import urlparse
 
-import yaml
-
 import sagemaker
 import sagemaker.local.data
 import sagemaker.local.utils
@@ -77,6 +76,15 @@ def __init__(self, instance_type, instance_count, image, sagemaker_session=None)
         """
         from sagemaker.local.local_session import LocalSession
 
+        # check if docker-compose is installed
+        if find_executable("docker-compose") is None:
+            raise ImportError(
+                "'docker-compose' is not installed. "
+                "Local Mode features will not work without docker-compose. "
+                "For more information on how to install 'docker-compose', please, see "
+                "https://docs.docker.com/compose/install/"
+            )
+
         self.sagemaker_session = sagemaker_session or LocalSession()
         self.instance_type = instance_type
         self.instance_count = instance_count
@@ -451,6 +459,13 @@ def _generate_compose_file(self, command, additional_volumes=None, additional_en
         }
 
         docker_compose_path = os.path.join(self.container_root, DOCKER_COMPOSE_FILENAME)
+
+        try:
+            import yaml
+        except ImportError as e:
+            logging.error(sagemaker.utils._module_import_error("yaml", "Local mode", "local"))
+            raise e
+
         yaml_content = yaml.dump(content, default_flow_style=False)
         logger.info("docker compose file: \n%s", yaml_content)
         with open(docker_compose_path, "w") as f:

src/sagemaker/local/local_session.py

@@ -14,10 +14,10 @@
 from __future__ import absolute_import
 
 import logging
+import os
 import platform
 
 import boto3
-import urllib3
 from botocore.exceptions import ClientError
 
 from sagemaker.local.image import _SageMakerContainer
@@ -29,7 +29,7 @@
     _LocalTransformJob,
 )
 from sagemaker.session import Session
-from sagemaker.utils import get_config_value
+from sagemaker.utils import get_config_value, _module_import_error
 
 logger = logging.getLogger(__name__)
 
@@ -323,6 +323,12 @@ def __init__(self, config=None):
             config (dict): Optional configuration for this client. In particular only
                 the local port is read.
         """
+        try:
+            import urllib3
+        except ImportError as e:
+            logging.error(_module_import_error("urllib3", "Local mode", "local"))
+            raise e
+
         self.http = urllib3.PoolManager()
         self.serving_port = 8080
         self.config = config
@@ -397,6 +403,16 @@ def _initialize(self, boto_session, sagemaker_client, sagemaker_runtime_client):
         self.sagemaker_runtime_client = LocalSagemakerRuntimeClient(self.config)
         self.local_mode = True
 
+        sagemaker_config_file = os.path.join(os.path.expanduser("~"), ".sagemaker", "config.yaml")
+        if os.path.exists(sagemaker_config_file):
+            try:
+                import yaml
+            except ImportError as e:
+                logging.error(_module_import_error("yaml", "Local mode", "local"))
+                raise e
+
+            self.config = yaml.load(open(sagemaker_config_file, "r"))
+
     def logs_for_job(self, job_name, wait=False, poll=5, log_type="All"):
         """
 

src/sagemaker/logs.py

@@ -1,4 +1,4 @@
-# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2017-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -31,7 +31,9 @@ class ColorWrap(object):
     cell).
     """
 
-    _stream_colors = [31, 32, 33, 34, 35, 36]
+    # For what color each number represents, see
+    # https://misc.flogisoft.com/bash/tip_colors_and_formatting#colors
+    _stream_colors = [34, 35, 32, 36, 33]
 
     def __init__(self, force=False):
         """Initialize the class.

src/sagemaker/session.py

@@ -25,7 +25,6 @@
 import boto3
 import botocore.config
 from botocore.exceptions import ClientError
-import yaml
 
 import sagemaker.logs
 from sagemaker import vpc_utils
@@ -95,11 +94,8 @@ def __init__(self, boto_session=None, sagemaker_client=None, sagemaker_runtime_c
         """
         self._default_bucket = None
 
-        sagemaker_config_file = os.path.join(os.path.expanduser("~"), ".sagemaker", "config.yaml")
-        if os.path.exists(sagemaker_config_file):
-            self.config = yaml.load(open(sagemaker_config_file, "r"))
-        else:
-            self.config = None
+        # currently is used for local_code in local mode
+        self.config = None
 
         self._initialize(boto_session, sagemaker_client, sagemaker_runtime_client)
 

src/sagemaker/utils.py

@@ -684,3 +684,23 @@ def __getattr__(self, name):
             name:
         """
         raise self.exc
+
+
+def _module_import_error(py_module, feature, extras):
+    """Return error message for module import errors, provide
+    installation details.
+
+    Args:
+        py_module (str): Module that failed to be imported
+        feature (str): Affected SageMaker feature
+        extras (str): Name of the `extras_require` to install the relevant dependencies
+
+    Returns:
+        str: Error message with installation instructions.
+    """
+    error_msg = (
+        "Failed to import {}. {} features will be impaired or broken. "
+        "Please run \"pip install 'sagemaker[{}]'\" "
+        "to install all required dependencies."
+    )
+    return error_msg.format(py_module, feature, extras)

tox.ini

@@ -119,6 +119,7 @@ deps =
 # pip install requirements.txt is separate as RTD does it in separate steps
 # having the requirements.txt installed in deps above results in Double Requirement exception
 # https://github.com/pypa/pip/issues/988
+# TODO: Remove pip install python-dateutil==2.8.0 once botocore no longer requires a pinned version
 commands =
     pip install python-dateutil==2.8.0
     pip install ../tests/data/smdebug_rulesconfig-0.4a0-py2.py3-none-any.whl