Merge remote-tracking branch 'public/master' into pytorch-release

Author: nadiaya

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+application_import_names = sagemaker, tests
+import-order-style = google

.gitignore

@@ -24,3 +24,4 @@ doc/_templates
 venv/
 *~
 .pytest_cache/
+*.swp

CHANGELOG.rst

@@ -2,13 +2,45 @@
 CHANGELOG
 =========
 
+1.4.1
+=====
+
+* bug-fix: Local Mode: Fix for non Framework containers
+
+1.4.0
+=====
+
+* bug-fix: Remove __all__ and add noqa in __init__
+* bug-fix: Estimators: Change max_iterations hyperparameter key for KMeans
+* bug-fix: Estimators: Remove unused argument job_details for ``EstimatorBase.attach()``
+* bug-fix: Local Mode: Show logs in Jupyter notebooks
+* feature: HyperparameterTuner: Add support for hyperparameter tuning jobs
+* feature: Analytics: Add functions for metrics in Training and Hyperparameter Tuning jobs
+* feature: Estimators: add support for tagging training jobs
+
+1.3.0
+=====
+
+* feature: Add chainer
+
+1.2.5
+=====
+
+* bug-fix: Change module names to string type in __all__
+* feature: Save training output files in local mode
+* bug-fix: tensorflow-serving-api: SageMaker does not conflict with tensorflow-serving-api module version
+* feature: Local Mode: add support for local training data using file://
+* feature: Updated TensorFlow Serving api protobuf files
+* bug-fix: No longer poll for logs from stopped training jobs
+
 1.2.4
 =====
 
 * feature: Estimators: add support for Amazon Random Cut Forest algorithm
 
 1.2.3
-=========
+=====
+
 * bug-fix: Fix local mode not using the right s3 bucket
 
 1.2.2

README.rst

@@ -1,4 +1,17 @@
-# -*- coding: utf-8 -*-
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
 import os
 import sys
 from datetime import datetime

doc/conf.py

@@ -1,7 +1,19 @@
-import os
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
 from glob import glob
-from os.path import basename
-from os.path import splitext
+import os
 
 from setuptools import setup, find_packages
 
@@ -11,11 +23,11 @@ def read(fname):
 
 
 setup(name="sagemaker",
-      version="1.3.dev",
+      version="1.5.dev",
       description="Open source library for training and deploying models on Amazon SageMaker.",
       packages=find_packages('src'),
       package_dir={'': 'src'},
-      py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')],
+      py_modules=[os.splitext(os.basename(path))[0] for path in glob('src/*.py')],
       long_description=read('README.rst'),
       author="Amazon Web Services",
       url='https://github.com/aws/sagemaker-python-sdk/',
@@ -37,7 +49,7 @@ def read(fname):
 
       extras_require={
           'test': ['tox', 'flake8', 'pytest', 'pytest-cov', 'pytest-xdist',
-                   'mock', 'tensorflow>=1.3.0', 'contextlib2', 'awslogs']},
+                   'mock', 'tensorflow>=1.3.0', 'contextlib2', 'awslogs', 'pandas']},
 
       entry_points={
           'console_scripts': ['sagemaker=sagemaker.cli.main:main'],

setup.py

@@ -1,4 +1,4 @@
-# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -12,31 +12,24 @@
 # language governing permissions and limitations under the License.
 from __future__ import absolute_import
 
-from sagemaker import estimator
-from sagemaker.amazon.kmeans import KMeans, KMeansModel, KMeansPredictor
-from sagemaker.amazon.pca import PCA, PCAModel, PCAPredictor
-from sagemaker.amazon.lda import LDA, LDAModel, LDAPredictor
-from sagemaker.amazon.linear_learner import LinearLearner, LinearLearnerModel, LinearLearnerPredictor
-from sagemaker.amazon.factorization_machines import FactorizationMachines, FactorizationMachinesModel
-from sagemaker.amazon.factorization_machines import FactorizationMachinesPredictor
-from sagemaker.amazon.ntm import NTM, NTMModel, NTMPredictor
-from sagemaker.amazon.randomcutforest import RandomCutForest, RandomCutForestModel, RandomCutForestPredictor
+from sagemaker import estimator  # noqa: F401
+from sagemaker.amazon.kmeans import KMeans, KMeansModel, KMeansPredictor  # noqa: F401
+from sagemaker.amazon.pca import PCA, PCAModel, PCAPredictor  # noqa: F401
+from sagemaker.amazon.lda import LDA, LDAModel, LDAPredictor  # noqa: F401
+from sagemaker.amazon.linear_learner import LinearLearner, LinearLearnerModel, LinearLearnerPredictor  # noqa: F401
+from sagemaker.amazon.factorization_machines import FactorizationMachines, FactorizationMachinesModel  # noqa: F401
+from sagemaker.amazon.factorization_machines import FactorizationMachinesPredictor  # noqa: F401
+from sagemaker.amazon.ntm import NTM, NTMModel, NTMPredictor  # noqa: F401
+from sagemaker.amazon.randomcutforest import (RandomCutForest, RandomCutForestModel,  # noqa: F401
+                                              RandomCutForestPredictor)
 
-from sagemaker.local.local_session import LocalSession
+from sagemaker.analytics import TrainingJobAnalytics, HyperparameterTuningJobAnalytics  # noqa: F401
+from sagemaker.local.local_session import LocalSession  # noqa: F401
 
-from sagemaker.model import Model
-from sagemaker.predictor import RealTimePredictor
-from sagemaker.session import Session
-from sagemaker.session import container_def
-from sagemaker.session import production_variant
-from sagemaker.session import s3_input
-from sagemaker.session import get_execution_role
-
-
-__all__ = [estimator, KMeans, KMeansModel, KMeansPredictor, PCA, PCAModel, PCAPredictor, LinearLearner,
-           LinearLearnerModel, LinearLearnerPredictor,
-           LDA, LDAModel, LDAPredictor,
-           FactorizationMachines, FactorizationMachinesModel, FactorizationMachinesPredictor,
-           RandomCutForest, RandomCutForestModel, RandomCutForestPredictor,
-           Model, NTM, NTMModel, NTMPredictor, RealTimePredictor, Session, LocalSession,
-           container_def, s3_input, production_variant, get_execution_role]
+from sagemaker.model import Model  # noqa: F401
+from sagemaker.predictor import RealTimePredictor  # noqa: F401
+from sagemaker.session import Session  # noqa: F401
+from sagemaker.session import container_def  # noqa: F401
+from sagemaker.session import production_variant  # noqa: F401
+from sagemaker.session import s3_input  # noqa: F401
+from sagemaker.session import get_execution_role  # noqa: F401

src/sagemaker/__init__.py

@@ -0,0 +1,48 @@
+
+===================================
+AWS SageMaker Estimators and Models
+===================================
+
+Amazon SageMaker provides several built-in machine learning algorithms that you can use for a variety of problem types.
+
+The full list of algorithms is available on the AWS website: https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html
+
+SageMaker Python SDK includes Estimator wrappers for the AWS K-means, Principal Components Analysis(PCA), Linear Learner, Factorization Machines, Latent Dirichlet Allocation(LDA), Neural Topic Model(NTM) and Random Cut Forest algorithms.
+
+Definition and usage
+~~~~~~~~~~~~~~~~~~~~
+Estimators that wrap Amazon's built-in algorithms define algorithm's hyperparameters with defaults. When a default is not possible you need to provide the value during construction, e.g.:
+
+- ``KMeans`` Estimator requires parameter ``k`` to define number of clusters
+- ``PCA`` Estimator requires parameter ``num_components`` to define number of principal components
+
+Interaction is identical as any other Estimators. There are additional details about how data is specified.
+
+Input data format
+^^^^^^^^^^^^^^^^^
+Please note that Amazon's built-in algorithms are working best with protobuf ``recordIO`` format.
+The data is expected to be available in S3 location and depending on algorithm it can handle dat in multiple data channels.
+
+This package offers support to prepare data into required fomrat and upload data to S3.
+Provided class ``RecordSet`` captures necessary details like S3 location, number of records, data channel and is expected as input parameter when calling ``fit()``.
+
+Function ``record_set`` is available on algorithms objects to make it simple to achieve the above.
+It takes 2D numpy array as input, uploads data to S3 and returns ``RecordSet`` objects. By default it uses ``train`` data channel and no labels but can be specified when called.
+
+Please find an example code snippet for illustration:
+
+.. code:: python
+
+    from sagemaker import PCA
+    pca_estimator = PCA(role='SageMakerRole', train_instance_count=1, train_instance_type='ml.m4.xlarge', num_components=3)
+
+    import numpy as np
+    records = pca_estimator.record_set(np.arange(10).reshape(2,5))
+
+    pca_estimator.fit(records)
+
+
+Predictions support
+~~~~~~~~~~~~~~~~~~~
+Calling inference on deployed Amazon's built-in algorithms requires specific input format. By default, this library creates a predictor that allows to use just numpy data.
+Data is converted so that ``application/x-recordio-protobuf`` input format is used. Received response is deserialized from the protobuf and provided as result from the ``predict`` call.

src/sagemaker/amazon/README.rst

@@ -1,4 +1,4 @@
-# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of

src/sagemaker/amazon/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -10,14 +10,16 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
 import json
 import logging
 import tempfile
 from six.moves.urllib.parse import urlparse
 from sagemaker.amazon import validation
 from sagemaker.amazon.hyperparameter import Hyperparameter as hp  # noqa
 from sagemaker.amazon.common import write_numpy_to_dense_tensor
-from sagemaker.estimator import EstimatorBase
+from sagemaker.estimator import EstimatorBase, _TrainingJob
 from sagemaker.session import s3_input
 from sagemaker.utils import sagemaker_timestamp
 
@@ -90,11 +92,38 @@ def _prepare_init_params_from_job_description(cls, job_details):
         del init_params['image']
         return init_params
 
-    def fit(self, records, mini_batch_size=None, **kwargs):
+    def _prepare_for_training(self, records, mini_batch_size=None, job_name=None):
+        """Set hyperparameters needed for training.
+
+        Args:
+            * records (:class:`~RecordSet`): The records to train this ``Estimator`` on.
+            * mini_batch_size (int or None): The size of each mini-batch to use when training. If ``None``, a
+                default value will be used.
+            * job_name (str): Name of the training job to be created. If not specified, one is generated,
+                using the base name given to the constructor if applicable.
+        """
+        super(AmazonAlgorithmEstimatorBase, self)._prepare_for_training(job_name=job_name)
+
+        feature_dim = None
+
+        if isinstance(records, list):
+            for record in records:
+                if record.channel == 'train':
+                    feature_dim = record.feature_dim
+                    break
+            if feature_dim is None:
+                raise ValueError('Must provide train channel.')
+        else:
+            feature_dim = records.feature_dim
+
+        self.feature_dim = feature_dim
+        self.mini_batch_size = mini_batch_size
+
+    def fit(self, records, mini_batch_size=None, wait=True, logs=True, job_name=None):
         """Fit this Estimator on serialized Record objects, stored in S3.
 
         ``records`` should be an instance of :class:`~RecordSet`. This defines a collection of
-        s3 data files to train this ``Estimator`` on.
+        S3 data files to train this ``Estimator`` on.
 
         Training data is expected to be encoded as dense or sparse vectors in the "values" feature
         on each Record. If the data is labeled, the label is expected to be encoded as a list of
@@ -108,15 +137,19 @@ def fit(self, records, mini_batch_size=None, **kwargs):
 
         Args:
             records (:class:`~RecordSet`): The records to train this ``Estimator`` on
-            mini_batch_size (int or None): The size of each mini-batch to use when training. If None, a
+            mini_batch_size (int or None): The size of each mini-batch to use when training. If ``None``, a
                 default value will be used.
+            wait (bool): Whether the call should wait until the job completes (default: True).
+            logs (bool): Whether to show the logs produced by the job.
+                Only meaningful when wait is True (default: True).
+            job_name (str): Training job name. If not specified, the estimator generates a default job name,
+                based on the training image name and current timestamp.
         """
-        self.feature_dim = records.feature_dim
-        self.mini_batch_size = mini_batch_size
+        self._prepare_for_training(records, job_name=job_name, mini_batch_size=mini_batch_size)
 
-        data = {records.channel: s3_input(records.s3_data, distribution='ShardedByS3Key',
-                                          s3_data_type=records.s3_data_type)}
-        super(AmazonAlgorithmEstimatorBase, self).fit(data, **kwargs)
+        self.latest_training_job = _TrainingJob.start_new(self, records)
+        if wait:
+            self.latest_training_job.wait(logs=logs)
 
     def record_set(self, train, labels=None, channel="train"):
         """Build a :class:`~RecordSet` from a numpy :class:`~ndarray` matrix and label vector.
@@ -178,6 +211,14 @@ def __repr__(self):
         """Return an unambiguous representation of this RecordSet"""
         return str((RecordSet, self.__dict__))
 
+    def data_channel(self):
+        """Return a dictionary to represent the training data in a channel for use with ``fit()``"""
+        return {self.channel: self.records_s3_input()}
+
+    def records_s3_input(self):
+        """Return a s3_input to represent the training data"""
+        return s3_input(self.s3_data, distribution='ShardedByS3Key', s3_data_type=self.s3_data_type)
+
 
 def _build_shards(num_shards, array):
     if num_shards < 1:
@@ -234,14 +275,16 @@ def registry(region_name, algorithm=None):
             "us-east-1": "382416733822",
             "us-east-2": "404615174143",
             "us-west-2": "174872318107",
-            "eu-west-1": "438346466558"
+            "eu-west-1": "438346466558",
+            "ap-northeast-1": "351501993468"
         }[region_name]
     elif algorithm in ["lda"]:
         account_id = {
             "us-east-1": "766337827248",
             "us-east-2": "999911452149",
             "us-west-2": "266724342769",
-            "eu-west-1": "999678624901"
+            "eu-west-1": "999678624901",
+            "ap-northeast-1": "258307448986"
         }[region_name]
     else:
         raise ValueError("Algorithm class:{} doesn't have mapping to account_id with images".format(algorithm))

src/sagemaker/amazon/amazon_estimator.py

@@ -1,4 +1,4 @@
-# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -10,6 +10,8 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
 import io
 import struct
 import sys

src/sagemaker/amazon/common.py

@@ -1,4 +1,4 @@
-# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -10,6 +10,8 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
 from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase, registry
 from sagemaker.amazon.common import numpy_to_record_serializer, record_deserializer
 from sagemaker.amazon.hyperparameter import Hyperparameter as hp  # noqa