Merge branch 'framework-gamma' of github.com:ChuyangDeng/sagemaker-python-sdk into framework-gamma

Author: Deng

CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## v1.50.16 (2020-02-26)
+
+### Bug Fixes and Other Changes
+
+ * use sagemaker_session when initializing Constraints and Statistics
+ * add sagemaker_session parameter to DataCaptureConfig
+ * make AutoML.deploy use self.sagemaker_session by default
+
+### Testing and Release Infrastructure
+
+ * unset region during integ tests
+ * use sagemaker_session fixture in all Airflow tests
+ * remove remaining TF legacy mode integ tests
+
+## v1.50.15 (2020-02-25)
+
+### Bug Fixes and Other Changes
+
+ * enable Neo integ tests
+
+## v1.50.14.post0 (2020-02-24)
+
+### Testing and Release Infrastructure
+
+ * remove TF framework mode notebooks from PR build
+ * don't create docker network for all integ tests
+
 ## v1.50.14 (2020-02-20)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-1.50.15.dev0
+1.50.17.dev0

buildspec.yml

@@ -16,7 +16,7 @@ phases:
 
       - start_time=`date +%s`
       - |
-        execute-command-if-has-matching-changes "tox -e py36 -- tests/integ -m \"not local_mode\" -n 512 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"
+        execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py36 -- tests/integ -m \"not local_mode\" -n 512 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"
       - ./ci-scripts/displaytime.sh 'py36 tests/integ' $start_time
 
   post_build:

src/sagemaker/automl/automl.py

@@ -216,8 +216,7 @@ def deploy(
                 created from it.
             sagemaker_session (sagemaker.session.Session): A SageMaker Session
                 object, used for SageMaker interactions (default: None). If not
-                specified, one is created using the default AWS configuration
-                chain.
+                specified, the one originally associated with the ``AutoML`` instance is used.
             name (str): The pipeline model name. If None, a default model name will
                 be selected on each ``deploy``.
             endpoint_name (str): The name of the endpoint to create (default:
@@ -248,6 +247,8 @@ def deploy(
                 If ``predictor_cls`` is specified, the invocation of ``self.predictor_cls`` on
                 the created endpoint name. Otherwise, ``None``.
         """
+        sagemaker_session = sagemaker_session or self.sagemaker_session
+
         if candidate is None:
             candidate_dict = self.best_candidate()
             candidate = CandidateEstimator(candidate_dict, sagemaker_session=sagemaker_session)

src/sagemaker/model.py

@@ -304,8 +304,8 @@ def compile(
                 https://docs.aws.amazon.com/sagemaker/latest/dg/API_OutputConfig.html.
             input_shape (dict): Specifies the name and shape of the expected
                 inputs for your trained model in json dictionary form, for
-                example: {'data':[1,3,1024,1024]}, or {'var1': [1,1,28,28],
-                'var2':[1,1,28,28]}
+                example: {'data': [1,3,1024,1024]}, or {'var1': [1,1,28,28],
+                'var2': [1,1,28,28]}
             output_path (str): Specifies where to store the compiled model
             role (str): Execution role
             tags (list[dict]): List of tags for labeling a compilation job. For
@@ -436,7 +436,8 @@ def deploy(
 
         compiled_model_suffix = "-".join(instance_type.split(".")[:-1])
         if self._is_compiled_model:
-            self.name += compiled_model_suffix
+            name_prefix = self.name or utils.name_from_image(self.image)
+            self.name = "{}{}".format(name_prefix, compiled_model_suffix)
 
         self._create_sagemaker_model(instance_type, accelerator_type, tags)
         production_variant = sagemaker.production_variant(

src/sagemaker/model_monitor/data_capture_config.py

@@ -41,6 +41,7 @@ def __init__(
         capture_options=None,
         csv_content_types=None,
         json_content_types=None,
+        sagemaker_session=None,
     ):
         """Initialize a DataCaptureConfig object for capturing data from Amazon SageMaker Endpoints.
 
@@ -56,14 +57,21 @@ def __init__(
                 which data to capture between request and response.
             csv_content_types ([str]): Optional. Default=["text/csv"].
             json_content_types([str]): Optional. Default=["application/json"].
-
+            sagemaker_session (sagemaker.session.Session): A SageMaker Session
+                object, used for SageMaker interactions (default: None). If not
+                specified, one is created using the default AWS configuration
+                chain.
         """
         self.enable_capture = enable_capture
         self.sampling_percentage = sampling_percentage
         self.destination_s3_uri = destination_s3_uri
         if self.destination_s3_uri is None:
+            sagemaker_session = sagemaker_session or Session()
             self.destination_s3_uri = os.path.join(
-                "s3://", Session().default_bucket(), _MODEL_MONITOR_S3_PATH, _DATA_CAPTURE_S3_PATH
+                "s3://",
+                sagemaker_session.default_bucket(),
+                _MODEL_MONITOR_S3_PATH,
+                _DATA_CAPTURE_S3_PATH,
             )
 
         self.kms_key_id = kms_key_id

src/sagemaker/model_monitor/model_monitoring.py

@@ -278,7 +278,7 @@ def create_monitoring_schedule(
         normalized_monitoring_output = self._normalize_monitoring_output(output=output)
 
         statistics_object, constraints_object = self._get_baseline_files(
-            statistics=statistics, constraints=constraints
+            statistics=statistics, constraints=constraints, sagemaker_session=self.sagemaker_session
         )
 
         statistics_s3_uri = None
@@ -402,7 +402,7 @@ def update_monitoring_schedule(
             }
 
         statistics_object, constraints_object = self._get_baseline_files(
-            statistics=statistics, constraints=constraints
+            statistics=statistics, constraints=constraints, sagemaker_session=self.sagemaker_session
         )
 
         statistics_s3_uri = None
@@ -781,7 +781,7 @@ def _generate_monitoring_schedule_name(self, schedule_name=None):
         return name_from_base(base=base_name)
 
     @staticmethod
-    def _get_baseline_files(statistics, constraints):
+    def _get_baseline_files(statistics, constraints, sagemaker_session=None):
         """Populates baseline values if possible.
 
         Args:
@@ -791,6 +791,9 @@ def _get_baseline_files(statistics, constraints):
             constraints (sagemaker.model_monitor.Constraints or str): The constraints object or str.
                 If none, this method will attempt to retrieve a previously baselined constraints
                 object.
+            sagemaker_session (sagemaker.session.Session): Session object which manages interactions
+                with Amazon SageMaker APIs and any other AWS services needed. If not specified, one
+                is created using the default AWS configuration chain.
 
         Returns:
             sagemaker.model_monitor.Statistics, sagemaker.model_monitor.Constraints: The Statistics
@@ -799,9 +802,13 @@ def _get_baseline_files(statistics, constraints):
 
         """
         if statistics is not None and isinstance(statistics, string_types):
-            statistics = Statistics.from_s3_uri(statistics_file_s3_uri=statistics)
+            statistics = Statistics.from_s3_uri(
+                statistics_file_s3_uri=statistics, sagemaker_session=sagemaker_session
+            )
         if constraints is not None and isinstance(constraints, string_types):
-            constraints = Constraints.from_s3_uri(constraints_file_s3_uri=constraints)
+            constraints = Constraints.from_s3_uri(
+                constraints_file_s3_uri=constraints, sagemaker_session=sagemaker_session
+            )
 
         return statistics, constraints
 
@@ -1240,7 +1247,7 @@ def create_monitoring_schedule(
         )
 
         statistics_object, constraints_object = self._get_baseline_files(
-            statistics=statistics, constraints=constraints
+            statistics=statistics, constraints=constraints, sagemaker_session=self.sagemaker_session
         )
 
         constraints_s3_uri = None
@@ -1386,7 +1393,7 @@ def update_monitoring_schedule(
         )
 
         statistics_object, constraints_object = self._get_baseline_files(
-            statistics=statistics, constraints=constraints
+            statistics=statistics, constraints=constraints, sagemaker_session=self.sagemaker_session
         )
 
         statistics_s3_uri = None
@@ -1829,6 +1836,7 @@ def baseline_statistics(self, file_name=STATISTICS_JSON_DEFAULT_FILE_NAME, kms_k
             return Statistics.from_s3_uri(
                 statistics_file_s3_uri=os.path.join(baselining_job_output_s3_path, file_name),
                 kms_key=kms_key,
+                sagemaker_session=self.sagemaker_session,
             )
         except ClientError as client_error:
             if client_error.response["Error"]["Code"] == "NoSuchKey":
@@ -1866,6 +1874,7 @@ def suggested_constraints(self, file_name=CONSTRAINTS_JSON_DEFAULT_FILE_NAME, km
             return Constraints.from_s3_uri(
                 constraints_file_s3_uri=os.path.join(baselining_job_output_s3_path, file_name),
                 kms_key=kms_key,
+                sagemaker_session=self.sagemaker_session,
             )
         except ClientError as client_error:
             if client_error.response["Error"]["Code"] == "NoSuchKey":
@@ -1981,6 +1990,7 @@ def statistics(self, file_name=STATISTICS_JSON_DEFAULT_FILE_NAME, kms_key=None):
             return Statistics.from_s3_uri(
                 statistics_file_s3_uri=os.path.join(baselining_job_output_s3_path, file_name),
                 kms_key=kms_key,
+                sagemaker_session=self.sagemaker_session,
             )
         except ClientError as client_error:
             if client_error.response["Error"]["Code"] == "NoSuchKey":
@@ -2022,6 +2032,7 @@ def constraint_violations(
                     baselining_job_output_s3_path, file_name
                 ),
                 kms_key=kms_key,
+                sagemaker_session=self.sagemaker_session,
             )
         except ClientError as client_error:
             if client_error.response["Error"]["Code"] == "NoSuchKey":

src/sagemaker/model_monitor/monitoring_files.py

@@ -69,7 +69,10 @@ def save(self, new_save_location_s3_uri=None):
             self.file_s3_uri = new_save_location_s3_uri
 
         return S3Uploader.upload_string_as_file_body(
-            body=json.dumps(self.body_dict), desired_s3_uri=self.file_s3_uri, kms_key=self.kms_key
+            body=json.dumps(self.body_dict),
+            desired_s3_uri=self.file_s3_uri,
+            kms_key=self.kms_key,
+            session=self.session,
         )
 
 
@@ -252,7 +255,10 @@ def from_s3_uri(cls, constraints_file_s3_uri, kms_key=None, sagemaker_session=No
             raise error
 
         return cls(
-            body_dict=body_dict, constraints_file_s3_uri=constraints_file_s3_uri, kms_key=kms_key
+            body_dict=body_dict,
+            constraints_file_s3_uri=constraints_file_s3_uri,
+            kms_key=kms_key,
+            sagemaker_session=sagemaker_session,
         )
 
     @classmethod

src/sagemaker/predictor.py

@@ -192,14 +192,22 @@ def enable_data_capture(self):
         to enable data capture. For a more customized experience, refer to
         update_data_capture_config, instead.
         """
-        self.update_data_capture_config(data_capture_config=DataCaptureConfig(enable_capture=True))
+        self.update_data_capture_config(
+            data_capture_config=DataCaptureConfig(
+                enable_capture=True, sagemaker_session=self.sagemaker_session
+            )
+        )
 
     def disable_data_capture(self):
         """Updates the DataCaptureConfig for the Predictor's associated Amazon SageMaker Endpoint
         to disable data capture. For a more customized experience, refer to
         update_data_capture_config, instead.
         """
-        self.update_data_capture_config(data_capture_config=DataCaptureConfig(enable_capture=False))
+        self.update_data_capture_config(
+            data_capture_config=DataCaptureConfig(
+                enable_capture=False, sagemaker_session=self.sagemaker_session
+            )
+        )
 
     def update_data_capture_config(self, data_capture_config):
         """Updates the DataCaptureConfig for the Predictor's associated Amazon SageMaker Endpoint

tests/conftest.py

@@ -286,7 +286,7 @@ def alternative_cpu_instance_type(sagemaker_session, request):
 
 @pytest.fixture(scope="session")
 def cpu_instance_family(cpu_instance_type):
-    "_".join(cpu_instance_type.split(".")[0:2])
+    return "_".join(cpu_instance_type.split(".")[0:2])
 
 
 def pytest_generate_tests(metafunc):