Merge branch 'reinvent' into latest

Author: nadiaya

.flake8

@@ -1,3 +1,5 @@
 [flake8]
 application_import_names = sagemaker, tests
 import-order-style = google
+per-file-ignores =
+    tests/unit/test_tuner.py: F405

buildspec.yml

@@ -3,6 +3,9 @@ version: 0.2
 phases:
   build:
     commands:
+      # TODO-reinvent-2019 [akarpur]: Remove this (adding internal boto models)
+      - aws configure add-model --service-model file://./tests/data/boto_models/sagemaker/2017-07-24/normal.json --service-name sagemaker
+
       - IGNORE_COVERAGE=-
 
       # run integration tests
@@ -16,7 +19,7 @@ phases:
       - start_time=`date +%s`
       - |
         if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
-          tox -e py36 -- tests/integ -m "not local_mode" -n 48  --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{"region_name": "us-east-1"}'
+          tox -e py36 -- tests/integ -m "not local_mode" -n 48  --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{"region_name": "us-east-2"}'
         fi
       - ./ci-scripts/displaytime.sh 'py36 tests/integ' $start_time
 

doc/analytics.rst

@@ -15,3 +15,8 @@ Analytics
     :members:
     :undoc-members:
     :show-inheritance:
+
+.. autoclass:: sagemaker.analytics.ExperimentAnalytics
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/conf.py

@@ -41,7 +41,6 @@ def __getattr__(cls, name):
     "tensorflow.python.framework",
     "tensorflow_serving",
     "tensorflow_serving.apis",
-    "numpy",
     "scipy",
     "scipy.sparse",
 ]

doc/index.rst

@@ -164,3 +164,5 @@ SageMaker APIs to export configurations for creating and managing Airflow workfl
     :maxdepth: 2
 
     sagemaker.workflow.airflow
+
+

src/sagemaker/__init__.py

@@ -50,6 +50,7 @@
 from sagemaker.model import Model, ModelPackage  # noqa: F401
 from sagemaker.pipeline import PipelineModel  # noqa: F401
 from sagemaker.predictor import RealTimePredictor  # noqa: F401
+from sagemaker.processing import Processor, ScriptProcessor  # noqa: F401
 from sagemaker.session import Session  # noqa: F401
 from sagemaker.session import container_def, pipeline_container_def  # noqa: F401
 from sagemaker.session import production_variant  # noqa: F401

src/sagemaker/amazon/amazon_estimator.py

@@ -27,8 +27,8 @@
 from sagemaker.model import NEO_IMAGE_ACCOUNT
 from sagemaker.session import s3_input
 from sagemaker.utils import sagemaker_timestamp, get_ecr_image_uri_prefix
+from sagemaker.xgboost.defaults import XGBOOST_VERSION_1, XGBOOST_SUPPORTED_VERSIONS
 from sagemaker.xgboost.estimator import get_xgboost_image_uri
-from sagemaker.xgboost.defaults import XGBOOST_LATEST_VERSION
 
 logger = logging.getLogger(__name__)
 
@@ -178,7 +178,15 @@ def _prepare_for_training(self, records, mini_batch_size=None, job_name=None):
         self.feature_dim = feature_dim
         self.mini_batch_size = mini_batch_size
 
-    def fit(self, records, mini_batch_size=None, wait=True, logs=True, job_name=None):
+    def fit(
+        self,
+        records,
+        mini_batch_size=None,
+        wait=True,
+        logs=True,
+        job_name=None,
+        experiment_config=None,
+    ):
         """Fit this Estimator on serialized Record objects, stored in S3.
 
         ``records`` should be an instance of :class:`~RecordSet`. This
@@ -206,10 +214,16 @@ def fit(self, records, mini_batch_size=None, wait=True, logs=True, job_name=None
             job_name (str): Training job name. If not specified, the estimator
                 generates a default job name, based on the training image name
                 and current timestamp.
+            experiment_config (dict[str, str]): Experiment management configuration.
+                Dictionary contains three optional keys, 'ExperimentName',
+                'TrialName', and 'TrialComponentName'
+                (default: ``None``).
         """
         self._prepare_for_training(records, job_name=job_name, mini_batch_size=mini_batch_size)
 
-        self.latest_training_job = _TrainingJob.start_new(self, records)
+        self.latest_training_job = _TrainingJob.start_new(
+            self, records, experiment_config=experiment_config
+        )
         if wait:
             self.latest_training_job.wait(logs=logs)
 
@@ -559,13 +573,23 @@ def get_image_uri(region_name, repo_name, repo_version=1):
     """
     if repo_name == "xgboost":
         if repo_version in ["0.90", "0.90-1", "0.90-1-cpu-py3"]:
-            return get_xgboost_image_uri(region_name, XGBOOST_LATEST_VERSION)
+            return get_xgboost_image_uri(region_name, XGBOOST_VERSION_1)
+
+        supported_version = [
+            version
+            for version in XGBOOST_SUPPORTED_VERSIONS
+            if repo_version in (version, version + "-cpu-py3")
+        ]
+        if supported_version:
+            return get_xgboost_image_uri(region_name, supported_version[0])
+
         logging.warning(
-            "There is a more up to date SageMaker XGBoost image."
+            "There is a more up to date SageMaker XGBoost image. "
             "To use the newer image, please set 'repo_version'="
-            "'0.90-1. For example:\n"
+            "'%s'. For example:\n"
             "\tget_image_uri(region, 'xgboost', '%s').",
-            XGBOOST_LATEST_VERSION,
+            XGBOOST_VERSION_1,
+            XGBOOST_VERSION_1,
         )
     repo = "{}:{}".format(repo_name, repo_version)
     return "{}/{}".format(registry(region_name, repo_name), repo)

src/sagemaker/amazon/linear_learner.py

@@ -366,7 +366,7 @@ def __init__(
         self.balance_multiclass_weights = balance_multiclass_weights
 
         if self.predictor_type == "multiclass_classifier" and (
-            num_classes is None or num_classes < 3
+            num_classes is None or int(num_classes) < 3
         ):
             raise ValueError(
                 "For predictor_type 'multiclass_classifier', 'num_classes' should be set to a "