Merge branch 'master' into master

Author: jesterhazy

.gitignore

@@ -19,3 +19,4 @@ examples/tensorflow/distributed_mnist/data
 *.iml
 doc/_build
 **/.DS_Store
+venv/

README.rst

@@ -1506,3 +1506,24 @@ Once you have the aws cli installed, you can upload a directory of files to S3 w
     aws s3 cp /tmp/foo/ s3://bucket/path
 
 You can read more about using the aws cli for manipulating S3 resources in the `AWS cli command reference <http://docs.aws.amazon.com/cli/latest/reference/s3/index.html>`__.
+
+
+How do I make predictions against an existing endpoint?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Create a Predictor object and provide it your endpoint name. Then, simply call its predict() method with your input.
+
+You can either use the generic RealTimePredictor class, which by default does not perform any serialization/deserialization transformations on your input, but can be configured to do so through constructor arguments:
+http://sagemaker.readthedocs.io/en/latest/predictors.html
+
+Or you can use the TensorFlow / MXNet specific predictor classes, which have default serialization/deserialization logic:
+http://sagemaker.readthedocs.io/en/latest/sagemaker.tensorflow.html#tensorflow-predictor
+http://sagemaker.readthedocs.io/en/latest/sagemaker.mxnet.html#mxnet-predictor
+
+Example code using the TensorFlow predictor:
+
+::
+
+    from sagemaker.tensorflow import TensorFlowPredictor
+    
+    predictor = TensorFlowPredictor('myexistingendpoint')
+    result = predictor.predict(['my request body'])

setup.py

@@ -32,7 +32,7 @@ def read(fname):
       ],
 
       # Declare minimal set for installation
-      install_requires=['boto3>=1.4.8', 'numpy>=1.9.0', 'protobuf>=3.1'],
+      install_requires=['boto3>=1.4.8', 'numpy>=1.9.0', 'protobuf>=3.1', 'scipy>=1.0.0'],
 
       extras_require={
           'test': ['tox', 'flake8', 'pytest', 'pytest-cov', 'pytest-xdist',

src/sagemaker/amazon/common.py

@@ -15,6 +15,7 @@
 import sys
 
 import numpy as np
+from scipy.sparse import issparse
 
 from sagemaker.amazon.record_pb2 import Record
 
@@ -64,6 +65,24 @@ def _write_label_tensor(resolved_type, record, scalar):
         record.label["values"].float32_tensor.values.extend([scalar])
 
 
+def _write_keys_tensor(resolved_type, record, vector):
+    if resolved_type == "Int32":
+        record.features["values"].int32_tensor.keys.extend(vector)
+    elif resolved_type == "Float64":
+        record.features["values"].float64_tensor.keys.extend(vector)
+    elif resolved_type == "Float32":
+        record.features["values"].float32_tensor.keys.extend(vector)
+
+
+def _write_shape(resolved_type, record, scalar):
+    if resolved_type == "Int32":
+        record.features["values"].int32_tensor.shape.extend([scalar])
+    elif resolved_type == "Float64":
+        record.features["values"].float64_tensor.shape.extend([scalar])
+    elif resolved_type == "Float32":
+        record.features["values"].float32_tensor.shape.extend([scalar])
+
+
 def write_numpy_to_dense_tensor(file, array, labels=None):
     """Writes a numpy array to a dense tensor"""
 
@@ -89,6 +108,46 @@ def write_numpy_to_dense_tensor(file, array, labels=None):
         _write_recordio(file, record.SerializeToString())
 
 
+def write_spmatrix_to_sparse_tensor(file, array, labels=None):
+    """Writes a scipy sparse matrix to a sparse tensor"""
+
+    if not issparse(array):
+        raise TypeError("Array must be sparse")
+
+    # Validate shape of array and labels, resolve array and label types
+    if not len(array.shape) == 2:
+        raise ValueError("Array must be a Matrix")
+    if labels is not None:
+        if not len(labels.shape) == 1:
+            raise ValueError("Labels must be a Vector")
+        if labels.shape[0] not in array.shape:
+            raise ValueError("Label shape {} not compatible with array shape {}".format(
+                             labels.shape, array.shape))
+        resolved_label_type = _resolve_type(labels.dtype)
+    resolved_type = _resolve_type(array.dtype)
+
+    csr_array = array.tocsr()
+    n_rows, n_cols = csr_array.shape
+
+    record = Record()
+    for row_idx in range(n_rows):
+        record.Clear()
+        row = csr_array.getrow(row_idx)
+        # Write values
+        _write_feature_tensor(resolved_type, record, row.data)
+        # Write keys
+        _write_keys_tensor(resolved_type, record, row.indices.astype(np.uint64))
+
+        # Write labels
+        if labels is not None:
+            _write_label_tensor(resolved_label_type, record, labels[row_idx])
+
+        # Write shape
+        _write_shape(resolved_type, record, n_cols)
+
+        _write_recordio(file, record.SerializeToString())
+
+
 def read_records(file):
     """Eagerly read a collection of amazon Record protobuf objects from file."""
     records = []

src/sagemaker/session.py

@@ -170,6 +170,13 @@ def default_bucket(self):
             elif error_code == 'OperationAborted' and 'conflicting conditional operation' in message:
                 # If this bucket is already being concurrently created, we don't need to create it again.
                 pass
+            elif error_code == 'TooManyBuckets':
+                # Succeed if the default bucket exists
+                try:
+                    s3.meta.client.head_bucket(Bucket=default_bucket)
+                    pass
+                except ClientError:
+                    raise
             else:
                 raise
 

tests/data/iris/failure_script.py

@@ -0,0 +1,3 @@
+def estimator_fn(run_config, params):
+    """For use with integration tests expecting failures."""
+    raise Exception('This failure is expected.')

tests/data/mxnet_mnist/failure_script.py

@@ -0,0 +1,3 @@
+def train(**kwargs):
+    """For use with integration tests expecting failures."""
+    raise Exception('This failure is expected.')

tests/integ/test_mxnet_train.py

@@ -70,3 +70,20 @@ def test_deploy_model(mxnet_training_job, sagemaker_session):
 
         data = numpy.zeros(shape=(1, 1, 28, 28))
         predictor.predict(data)
+
+
+def test_failed_training_job(sagemaker_session):
+    with timeout(minutes=15):
+        script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'failure_script.py')
+        data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
+
+        mx = MXNet(entry_point=script_path, role='SageMakerRole',
+                   train_instance_count=1, train_instance_type='ml.c4.xlarge',
+                   sagemaker_session=sagemaker_session)
+
+        train_input = mx.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
+                                                       key_prefix='integ-test-data/mxnet_mnist/train-failure')
+
+        with pytest.raises(ValueError) as e:
+            mx.fit(train_input)
+        assert 'This failure is expected' in str(e.value)

tests/integ/test_tf.py

@@ -19,6 +19,8 @@
 from tests.integ import DATA_DIR, REGION
 from tests.integ.timeout import timeout_and_delete_endpoint, timeout
 
+DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data')
+
 
 @pytest.fixture(scope='module')
 def sagemaker_session():
@@ -28,7 +30,6 @@ def sagemaker_session():
 def test_tf(sagemaker_session):
     with timeout(minutes=15):
         script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py')
-        data_path = os.path.join(DATA_DIR, 'iris', 'data')
 
         estimator = TensorFlow(entry_point=script_path,
                                role='SageMakerRole',
@@ -40,7 +41,7 @@ def test_tf(sagemaker_session):
                                sagemaker_session=sagemaker_session,
                                base_job_name='test-tf')
 
-        inputs = estimator.sagemaker_session.upload_data(path=data_path, key_prefix='integ-test-data/tf_iris')
+        inputs = estimator.sagemaker_session.upload_data(path=DATA_PATH, key_prefix='integ-test-data/tf_iris')
         estimator.fit(inputs)
         print('job succeeded: {}'.format(estimator.latest_training_job.name))
 
@@ -49,3 +50,22 @@ def test_tf(sagemaker_session):
 
         result = json_predictor.predict([6.4, 3.2, 4.5, 1.5])
         print('predict result: {}'.format(result))
+
+
+def test_failed_tf_training(sagemaker_session):
+    with timeout(minutes=15):
+        script_path = os.path.join(DATA_DIR, 'iris', 'failure_script.py')
+        estimator = TensorFlow(entry_point=script_path,
+                               role='SageMakerRole',
+                               training_steps=1,
+                               evaluation_steps=1,
+                               hyperparameters={'input_tensor_name': 'inputs'},
+                               train_instance_count=1,
+                               train_instance_type='ml.c4.xlarge',
+                               sagemaker_session=sagemaker_session)
+
+        inputs = estimator.sagemaker_session.upload_data(path=DATA_PATH, key_prefix='integ-test-data/tf-failure')
+
+        with pytest.raises(ValueError) as e:
+            estimator.fit(inputs)
+        assert 'This failure is expected' in str(e.value)