Merge branch 'master' into local_pull

Author: ChoiByungWook

.pylintrc

@@ -0,0 +1,90 @@
+[MASTER]
+
+ignore=
+    tensorflow_serving
+
+[MESSAGES CONTROL]
+
+disable=
+    C, # convention
+    R, # refactor
+    too-many-arguments, # We should fix the offending ones soon.
+    too-many-lines, # Some files are too big, we should fix this too
+    too-few-public-methods,
+    too-many-instance-attributes,
+    too-many-locals,
+    len-as-condition, # Nice to have in the future
+    bad-indentation,
+    line-too-long, # We let Flake8 take care of this
+    logging-format-interpolation,
+    useless-object-inheritance, # We still support python2 so inheriting from object is ok
+    invalid-name,
+    import-error,
+    logging-not-lazy,
+    fixme,
+    no-self-use,
+    attribute-defined-outside-init,
+    protected-access,
+    invalid-all-object,
+    arguments-differ,
+    abstract-method,
+    signature-differs
+
+[REPORTS]
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html
+output-format=colorized
+
+# Tells whether to display a full report or only the messages
+# CHANGE: No report.
+reports=no
+
+[FORMAT]
+# Maximum number of characters on a single line.
+max-line-length=100
+# Maximum number of lines in a module
+#max-module-lines=1000
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 tab).
+indent-string='  '
+
+[BASIC]
+
+# Required attributes for module, separated by a comma
+#required-attributes=
+# List of builtins function names that should not be used, separated by a comma.
+# XXX: Should we ban map() & filter() for list comprehensions?
+# exit & quit are for the interactive interpreter shell only.
+# https://docs.python.org/3/library/constants.html#constants-added-by-the-site-module
+bad-functions=
+    apply,
+    exit,
+    input,
+    quit,
+
+[SIMILARITIES]
+# Minimum lines number of a similarity.
+min-similarity-lines=5
+# Ignore comments when computing similarities.
+ignore-comments=yes
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+[VARIABLES]
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+# A regular expression matching the beginning of the name of dummy variables
+# (i.e. not used).
+dummy-variables-rgx=_|unused_
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+#additional-builtins=
+
+[LOGGING]
+# Apply logging string format checks to calls on these modules.
+logging-modules=
+    logging
+
+[TYPECHECK]
+ignored-modules=
+    distutils

CHANGELOG.rst

@@ -2,11 +2,44 @@
 CHANGELOG
 =========
 
-1.13.1dev
+1.14.3-dev
 ======
 
+* bug-fix: Changes to use correct S3 bucket and time range for dataframes in TrainingJobAnalytics.
+* bug-fix: Local Mode: correctly handle the case where the model output folder doesn't exist yet
+* feature: Add APIs to export Airflow training and tuning config
+* doc-fix: Fix typos in tensorflow serving documentation
+* doc-fix: Add estimator base classes to API docs
+* feature: HyperparameterTuner: add support for Automatic Model Tuning's Warm Start Jobs
 * enhancement: Local Mode: add explicit pull for serving
 
+1.14.2
+======
+
+* bug-fix: support ``CustomAttributes`` argument in local mode ``invoke_endpoint`` requests
+* enhancement: add ``content_type`` parameter to ``sagemaker.tensorflow.serving.Predictor``
+* doc-fix: add TensorFlow Serving Container docs
+* doc-fix: fix rendering error in README.rst
+* enhancement: Local Mode: support optional input channels
+* build: added pylint
+* build: upgrade docker-compose to 1.23
+* enhancement: Frameworks: update warning for not setting framework_version as we aren't planning a breaking change anymore
+* enhancement: Session: remove hardcoded 'training' from job status error message
+* bug-fix: Updated Cloudwatch namespace for metrics in TrainingJobsAnalytics
+* bug-fix: Changes to use correct s3 bucket and time range for dataframes in TrainingJobAnalytics.
+* enhancement: Remove MetricDefinition lookup via tuning job in TrainingJobAnalytics
+
+1.14.1
+======
+
+* feature: Estimators: add support for Amazon Object2Vec algorithm
+
+1.14.0
+======
+
+* feature: add support for sagemaker-tensorflow-serving container
+* feature: Estimator: make input channels optional
+
 1.13.0
 ======
 

README.rst

@@ -290,6 +290,7 @@ If your bring-your-own case requires different settings, you can create your own
 Here's an example of how to use incremental training:
 
 .. code:: python
+
     # Configure an estimator
     estimator = sagemaker.estimator.Estimator(training_image,
                                               role,
@@ -394,7 +395,7 @@ Amazon SageMaker provides several built-in machine learning algorithms that you
 The full list of algorithms is available at: https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html
 
 The SageMaker Python SDK includes estimator wrappers for the AWS K-means, Principal Components Analysis (PCA), Linear Learner, Factorization Machines,
-Latent Dirichlet Allocation (LDA), Neural Topic Model (NTM) Random Cut Forest and k-nearest neighbors (k-NN) algorithms.
+Latent Dirichlet Allocation (LDA), Neural Topic Model (NTM), Random Cut Forest, k-nearest neighbors (k-NN), and Object2Vec algorithms.
 
 For more information, see `AWS SageMaker Estimators and Models`_.
 

doc/conf.py

@@ -32,7 +32,7 @@ def __getattr__(cls, name):
                 'numpy', 'scipy', 'scipy.sparse']
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
-version = '1.11.3'
+version = '1.14.0'
 project = u'sagemaker'
 
 # Add any Sphinx extension module names here, as strings. They can be extensions

doc/estimators.rst

@@ -1,10 +1,22 @@
 Estimators
---------------------
+----------
 
 A high level interface for SageMaker training
 
+.. autoclass:: sagemaker.estimator.EstimatorBase
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:
+
 .. autoclass:: sagemaker.estimator.Estimator
     :members:
     :undoc-members:
     :show-inheritance:
     :inherited-members:
+
+.. autoclass:: sagemaker.estimator.Framework
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

setup.py

@@ -53,9 +53,9 @@ def read(fname):
       ],
 
       # Declare minimal set for installation
-      install_requires=['boto3>=1.4.8', 'numpy>=1.9.0', 'protobuf>=3.1', 'scipy>=0.19.0',
-                        'urllib3 >=1.21, <1.23',
-                        'PyYAML>=3.2', 'protobuf3-to-dict>=0.1.5', 'docker-compose>=1.21.0'],
+      install_requires=['boto3>=1.9.38', 'numpy>=1.9.0', 'protobuf>=3.1', 'scipy>=0.19.0',
+                        'urllib3 >=1.21', 'PyYAML>=3.2', 'protobuf3-to-dict>=0.1.5',
+                        'docker-compose>=1.23.0'],
 
       extras_require={
           'test': ['tox', 'flake8', 'pytest', 'pytest-cov', 'pytest-xdist',

src/sagemaker/__init__.py

@@ -23,6 +23,7 @@
 from sagemaker.amazon.randomcutforest import (RandomCutForest, RandomCutForestModel,  # noqa: F401
                                               RandomCutForestPredictor)
 from sagemaker.amazon.knn import KNN, KNNModel, KNNPredictor  # noqa: F401
+from sagemaker.amazon.object2vec import Object2Vec, Object2VecModel  # noqa: F401
 
 from sagemaker.analytics import TrainingJobAnalytics, HyperparameterTuningJobAnalytics  # noqa: F401
 from sagemaker.local.local_session import LocalSession  # noqa: F401
@@ -35,4 +36,4 @@
 from sagemaker.session import s3_input  # noqa: F401
 from sagemaker.session import get_execution_role  # noqa: F401
 
-__version__ = '1.13.0'
+__version__ = '1.14.2'

src/sagemaker/amazon/README.rst

@@ -7,7 +7,7 @@ Amazon SageMaker provides several built-in machine learning algorithms that you
 
 The full list of algorithms is available on the AWS website: https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html
 
-SageMaker Python SDK includes Estimator wrappers for the AWS K-means, Principal Components Analysis(PCA), Linear Learner, Factorization Machines, Latent Dirichlet Allocation(LDA), Neural Topic Model(NTM), Random Cut Forest algorithms and k-nearest neighbors (k-NN).
+SageMaker Python SDK includes Estimator wrappers for the AWS K-means, Principal Components Analysis(PCA), Linear Learner, Factorization Machines, Latent Dirichlet Allocation(LDA), Neural Topic Model(NTM), Random Cut Forest algorithms, k-nearest neighbors (k-NN) and Object2Vec.
 
 Definition and usage
 ~~~~~~~~~~~~~~~~~~~~

src/sagemaker/amazon/amazon_estimator.py

@@ -15,7 +15,9 @@
 import json
 import logging
 import tempfile
+
 from six.moves.urllib.parse import urlparse
+
 from sagemaker.amazon import validation
 from sagemaker.amazon.hyperparameter import Hyperparameter as hp  # noqa
 from sagemaker.amazon.common import write_numpy_to_dense_tensor
@@ -32,6 +34,8 @@ class AmazonAlgorithmEstimatorBase(EstimatorBase):
 
     feature_dim = hp('feature_dim', validation.gt(0), data_type=int)
     mini_batch_size = hp('mini_batch_size', validation.gt(0), data_type=int)
+    repo_name = None
+    repo_version = None
 
     def __init__(self, role, train_instance_count, train_instance_type, data_location=None, **kwargs):
         """Initialize an AmazonAlgorithmEstimatorBase.
@@ -263,7 +267,7 @@ def upload_numpy_to_s3_shards(num_shards, s3, bucket, key_prefix, array, labels=
             [{'prefix': 's3://{}/{}'.format(bucket, key_prefix)}] + uploaded_files)
         s3.Object(bucket, manifest_key).put(Body=manifest_str.encode('utf-8'))
         return "s3://{}/{}".format(bucket, manifest_key)
-    except Exception as ex:
+    except Exception as ex:  # pylint: disable=broad-except
         try:
             for file in uploaded_files:
                 s3.Object(bucket, key_prefix + file).delete()
@@ -280,7 +284,7 @@ def registry(region_name, algorithm=None):
     https://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/amazon
     """
     if algorithm in [None, "pca", "kmeans", "linear-learner", "factorization-machines", "ntm",
-                     "randomcutforest", "knn"]:
+                     "randomcutforest", "knn", "object2vec"]:
         account_id = {
             "us-east-1": "382416733822",
             "us-east-2": "404615174143",

src/sagemaker/amazon/kmeans.py

@@ -121,9 +121,9 @@ def _prepare_for_training(self, records, mini_batch_size=5000, job_name=None):
 
     def hyperparameters(self):
         """Return the SageMaker hyperparameters for training this KMeans Estimator"""
-        hp = dict(force_dense='True')  # KMeans requires this hp to fit on Record objects
-        hp.update(super(KMeans, self).hyperparameters())
-        return hp
+        hp_dict = dict(force_dense='True')  # KMeans requires this hp to fit on Record objects
+        hp_dict.update(super(KMeans, self).hyperparameters())
+        return hp_dict
 
 
 class KMeansPredictor(RealTimePredictor):