Merge branch 'master' into from-processing

Author: ajaykarpur

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v1.48.1 (2019-12-18)
+
+### Bug fixes and other changes
+
+ * use name_from_base in auto_ml.py but unique_name_from_base in tests.
+ * make test's custom bucket include region and account name.
+ * add Keras to the list of Neo-supported frameworks
+
+### Documentation changes
+
+ * add link to parent classes to clarify **kwargs
+ * add link to framework-related parent classes to clarify **kwargs
+
+## v1.48.0 (2019-12-17)
+
+### Features
+
+ * allow setting the default bucket in Session
+
+### Bug fixes and other changes
+
+ * set integration test parallelization to 512
+ * shorten base job name to avoid collision
+ * multi model integration test to create ECR repo with unique names to allow independent parallel executions
+
 ## v1.47.1 (2019-12-16)
 
 ### Bug fixes and other changes

VERSION

@@ -1 +1 @@
-1.47.2.dev0
+1.48.2.dev0

buildspec.yml

@@ -20,7 +20,7 @@ phases:
       - start_time=`date +%s`
       - |
         if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
-          tox -e py36 -- tests/integ -m "not local_mode" -n 48  --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{"region_name": "us-east-2"}'
+          tox -e py36 -- tests/integ -m "not local_mode" -n 512 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{"region_name": "us-east-2"}'
         fi
       - ./ci-scripts/displaytime.sh 'py36 tests/integ' $start_time
 

src/sagemaker/algorithm.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -59,110 +59,72 @@ def __init__(
 
         Args:
             algorithm_arn (str): algorithm arn used for training. Can be just the name if your
-                    account owns the algorithm.
-
-                role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker
-                    training jobs and APIsthat create Amazon SageMaker endpoints use this role to
-                    access training data and model artifacts. After the endpoint
-                    is created, the inference code might use the IAM role, if it
-                    needs to access an AWS resource.
-
-                train_instance_count (int): Number of Amazon EC2 instances to
+                account owns the algorithm.
+            role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker
+                training jobs and APIsthat create Amazon SageMaker endpoints use this role to
+                access training data and model artifacts. After the endpoint
+                is created, the inference code might use the IAM role, if it
+                needs to access an AWS resource.
+            train_instance_count (int): Number of Amazon EC2 instances to
                 use for training. train_instance_type (str): Type of EC2
                 instance to use for training, for example, 'ml.c4.xlarge'.
-                train_volume_size (int): Size in GB of the EBS volume to use for
-                storing input data
-
-                    during training (default: 30). Must be large enough to store
-                    training data if File Mode is used (which is the default).
-
-                train_volume_kms_key (str): Optional. KMS key ID for encrypting EBS volume attached
-                    to the training instance (default: None).
-
-                train_max_run (int): Timeout in seconds for training (default: 24 * 60 * 60).
-                    After this amount of time Amazon SageMaker terminates the
-                    job regardless of its current status.
-
-                input_mode (str): The input mode that the algorithm supports
-                (default: 'File'). Valid modes:
-
-                    * 'File' - Amazon SageMaker copies the training dataset from
-                      the S3 location to a local directory.
-                    * 'Pipe' - Amazon SageMaker streams data directly from S3 to
-                      the container via a Unix-named pipe.
-
-                    This argument can be overriden on a per-channel basis using
-                    ``sagemaker.session.s3_input.input_mode``.
-
-                output_path (str): S3 location for saving the training result (model artifacts and
-                    output files). If not specified, results are stored to a default bucket. If
-                    the bucket with the specific name does not exist, the
-                    estimator creates the bucket during the
-                    :meth:`~sagemaker.estimator.EstimatorBase.fit` method
-                    execution.
-
-                output_kms_key (str): Optional. KMS key ID for encrypting the
+            train_volume_size (int): Size in GB of the EBS volume to use for
+                storing input data during training (default: 30). Must be large enough to store
+                training data if File Mode is used (which is the default).
+            train_volume_kms_key (str): Optional. KMS key ID for encrypting EBS volume attached
+                to the training instance (default: None).
+            train_max_run (int): Timeout in seconds for training (default: 24 * 60 * 60).
+                After this amount of time Amazon SageMaker terminates the
+                job regardless of its current status.
+            input_mode (str): The input mode that the algorithm supports
+            (default: 'File'). Valid modes:
+
+                * 'File' - Amazon SageMaker copies the training dataset from
+                  the S3 location to a local directory.
+                * 'Pipe' - Amazon SageMaker streams data directly from S3 to
+                  the container via a Unix-named pipe.
+
+                This argument can be overriden on a per-channel basis using
+                ``sagemaker.session.s3_input.input_mode``.
+
+            output_path (str): S3 location for saving the training result (model artifacts and
+                output files). If not specified, results are stored to a default bucket. If
+                the bucket with the specific name does not exist, the
+                estimator creates the bucket during the
+                :meth:`~sagemaker.estimator.EstimatorBase.fit` method
+                execution.
+            output_kms_key (str): Optional. KMS key ID for encrypting the
                 training output (default: None). base_job_name (str): Prefix for
                 training job name when the
                 :meth:`~sagemaker.estimator.EstimatorBase.fit`
-
-                    method launches. If not specified, the estimator generates a
-                    default job name, based on the training image name and
-                    current timestamp.
-
-                sagemaker_session (sagemaker.session.Session): Session object which manages
-                    interactions with Amazon SageMaker APIs and any other AWS services needed. If
-                    not specified, the estimator creates one using the default
-                    AWS configuration chain.
-
-                tags (list[dict]): List of tags for labeling a training job. For more, see
-                    https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
-
-                subnets (list[str]): List of subnet ids. If not specified
+                method launches. If not specified, the estimator generates a
+                default job name, based on the training image name and
+                current timestamp.
+            sagemaker_session (sagemaker.session.Session): Session object which manages
+                interactions with Amazon SageMaker APIs and any other AWS services needed. If
+                not specified, the estimator creates one using the default
+                AWS configuration chain.
+            tags (list[dict]): List of tags for labeling a training job. For more, see
+                https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
+            subnets (list[str]): List of subnet ids. If not specified
                 training job will be created without VPC config.
                 security_group_ids (list[str]): List of security group ids. If
-                not specified training job will be created
-
-                    without VPC config.
-
-                model_uri (str): URI where a pre-trained model is stored, either locally or in S3
-                    (default: None). If specified, the estimator will create a channel pointing to
-                    the model so the training job can download it. This model
-                    can be a 'model.tar.gz' from a previous training job, or
-                    other artifacts coming from a different source.
-                    More information:
-                    https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html#td-deserialization
-
-                model_channel_name (str): Name of the channel where 'model_uri'
+                not specified training job will be created without VPC config.
+            model_uri (str): URI where a pre-trained model is stored, either locally or in S3
+                (default: None). If specified, the estimator will create a channel pointing to
+                the model so the training job can download it. This model
+                can be a 'model.tar.gz' from a previous training job, or
+                other artifacts coming from a different source.
+                More information:
+                https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html#td-deserialization
+            model_channel_name (str): Name of the channel where 'model_uri'
                 will be downloaded (default: 'model'). metric_definitions
                 (list[dict]): A list of dictionaries that defines the metric(s)
-                used to evaluate the
-
-                    training jobs. Each dictionary contains two keys: 'Name' for
-                    the name of the metric, and 'Regex' for the regular
-                    expression used to extract the metric from the logs.
-
-                encrypt_inter_container_traffic (bool): Specifies whether traffic between training
-                    containers is encrypted for the training job (default: ``False``).
-            role:
-            train_instance_count:
-            train_instance_type:
-            train_volume_size:
-            train_volume_kms_key:
-            train_max_run:
-            input_mode:
-            output_path:
-            output_kms_key:
-            base_job_name:
-            sagemaker_session:
-            hyperparameters:
-            tags:
-            subnets:
-            security_group_ids:
-            model_uri:
-            model_channel_name:
-            metric_definitions:
-            encrypt_inter_container_traffic:
+                used to evaluate the training jobs. Each dictionary contains two keys: 'Name' for
+                the name of the metric, and 'Regex' for the regular
+                expression used to extract the metric from the logs.
+            encrypt_inter_container_traffic (bool): Specifies whether traffic between training
+                containers is encrypted for the training job (default: ``False``).
             **kwargs: Additional kwargs. This is unused. It's only added for AlgorithmEstimator
                 to ignore the irrelevant arguments.
         """
@@ -309,7 +271,13 @@ def create_model(
                 the model. Default: use subnets and security groups from this Estimator.
                 * 'Subnets' (list[str]): List of subnet ids.
                 * 'SecurityGroupIds' (list[str]): List of security group ids.
-            **kwargs:
+            **kwargs: Additional arguments for creating a :class:`~sagemaker.model.ModelPackage`.
+
+        .. tip::
+
+            You can find additional parameters for using this method at
+            :class:`~sagemaker.model.ModelPackage` and
+            :class:`~sagemaker.model.Model`.
 
         Returns:
             a Model ready for deployment.

src/sagemaker/automl/automl.py

@@ -379,7 +379,7 @@ def _prepare_for_auto_ml_job(self, job_name=None):
             if self.base_job_name:
                 base_name = self.base_job_name
             else:
-                base_name = "sagemaker-auto-ml"
+                base_name = "automl"
             # CreateAutoMLJob API validates that member length less than or equal to 32
             self.current_job_name = name_from_base(base_name, max_length=32)
 

src/sagemaker/chainer/estimator.py

@@ -111,13 +111,20 @@ def __init__(
             image_name (str): If specified, the estimator will use this image
                 for training and hosting, instead of selecting the appropriate
                 SageMaker official image based on framework_version and
-                py_version. It can be an ECR url or dockerhub image and tag. ..
-                admonition:: Examples
+                py_version. It can be an ECR url or dockerhub image and tag.
+
+                Examples
+                    * ``123412341234.dkr.ecr.us-west-2.amazonaws.com/my-custom-image:1.0``
+                    * ``custom-image:latest``
 
-                    123.dkr.ecr.us-west-2.amazonaws.com/my-custom-image:1.0
-                    custom-image:latest.
             **kwargs: Additional kwargs passed to the
                 :class:`~sagemaker.estimator.Framework` constructor.
+
+        .. tip::
+
+            You can find additional parameters for initializing this class at
+            :class:`~sagemaker.estimator.Framework` and
+            :class:`~sagemaker.estimator.EstimatorBase`.
         """
         if framework_version is None:
             logger.warning(empty_framework_version_warning(CHAINER_VERSION, self.LATEST_VERSION))

src/sagemaker/chainer/model.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -93,8 +93,14 @@ def __init__(
             model_server_workers (int): Optional. The number of worker processes
                 used by the inference server. If None, server will use one
                 worker per vCPU.
-            **kwargs: Keyword arguments passed to the ``FrameworkModel``
-                initializer.
+            **kwargs: Keyword arguments passed to the
+                :class:`~sagemaker.model.FrameworkModel` initializer.
+
+        .. tip::
+
+            You can find additional parameters for initializing this class at
+            :class:`~sagemaker.model.FrameworkModel` and
+            :class:`~sagemaker.model.Model`.
         """
         super(ChainerModel, self).__init__(
             model_data, image, role, entry_point, predictor_cls=predictor_cls, **kwargs

src/sagemaker/estimator.py

@@ -492,8 +492,8 @@ def compile_model(
                 'var2':[1,1,28,28]}
             output_path (str): Specifies where to store the compiled model
             framework (str): The framework that is used to train the original
-                model. Allowed values: 'mxnet', 'tensorflow', 'pytorch', 'onnx',
-                'xgboost'
+                model. Allowed values: 'mxnet', 'tensorflow', 'keras', 'pytorch',
+                'onnx', 'xgboost'
             framework_version (str): The version of the framework
             compile_max_run (int): Timeout in seconds for compilation (default:
                 3 * 60). After this amount of time Amazon SageMaker Neo
@@ -1336,9 +1336,15 @@ def create_model(
                 Default: use subnets and security groups from this Estimator.
                 * 'Subnets' (list[str]): List of subnet ids.
                 * 'SecurityGroupIds' (list[str]): List of security group ids.
-            **kwargs:
+            **kwargs: Additional parameters passed to :class:`~sagemaker.model.Model`
+
+        .. tip::
+
+            You can find additional parameters for using this method at
+            :class:`~sagemaker.model.Model`.
 
-        Returns: a Model ready for deployment.
+        Returns:
+            (sagemaker.model.Model) a Model ready for deployment.
         """
         if predictor_cls is None:
 
@@ -1566,6 +1572,11 @@ def __init__(
                 (default: ``None``).
             **kwargs: Additional kwargs passed to the ``EstimatorBase``
                 constructor.
+
+        .. tip::
+
+            You can find additional parameters for initializing this class at
+            :class:`~sagemaker.estimator.EstimatorBase`.
         """
         super(Framework, self).__init__(**kwargs)
         if entry_point.startswith("s3://"):

src/sagemaker/model.py

@@ -45,7 +45,7 @@
         "qcs605",
     ]
 )
-NEO_ALLOWED_FRAMEWORKS = set(["mxnet", "tensorflow", "pytorch", "onnx", "xgboost"])
+NEO_ALLOWED_FRAMEWORKS = set(["mxnet", "tensorflow", "keras", "pytorch", "onnx", "xgboost"])
 
 NEO_IMAGE_ACCOUNT = {
     "us-west-1": "710691900526",
@@ -322,8 +322,8 @@ def compile(
                 3 * 60). After this amount of time Amazon SageMaker Neo
                 terminates the compilation job regardless of its current status.
             framework (str): The framework that is used to train the original
-                model. Allowed values: 'mxnet', 'tensorflow', 'pytorch', 'onnx',
-                'xgboost'
+                model. Allowed values: 'mxnet', 'tensorflow', 'keras', 'pytorch',
+                'onnx', 'xgboost'
             framework_version (str):
 
         Returns:
@@ -734,6 +734,11 @@ def __init__(
                 try to use either CodeCommit credential helper or local
                 credential storage for authentication.
             **kwargs: Keyword arguments passed to the ``Model`` initializer.
+
+        .. tip::
+
+            You can find additional parameters for initializing this class at
+            :class:`~sagemaker.model.Model`.
         """
         super(FrameworkModel, self).__init__(
             model_data,

src/sagemaker/multidatamodel.py

@@ -66,7 +66,13 @@ def __init__(
                 object, used for SageMaker interactions (default: None). If not
                 specified, one is created using the default AWS configuration
                 chain.
-            **kwargs: Keyword arguments passed to the ``Model`` initializer.
+            **kwargs: Keyword arguments passed to the
+                :class:`~sagemaker.model.Model` initializer.
+
+        .. tip::
+
+            You can find additional parameters for initializing this class at
+            :class:`~sagemaker.model.Model`.
         """
         # Validate path
         if not model_data_prefix.startswith("s3://"):