Merge branch 'master' into ci-queueing-improvements

Author: mufaddal-rohawala

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v2.66.2.post0 (2021-10-28)
+
+### Documentation Changes
+
+ * Update estimator docstrings to add Fast File Mode
+
+## v2.66.2 (2021-10-27)
+
+### Bug Fixes and Other Changes
+
+ * expose num_clusters parameter for clarify shap in shapconfig
+ * Update cron job to run hourly
+
 ## v2.66.1 (2021-10-26)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.66.2.dev0
+2.66.3.dev0

doc/api/inference/model.rst

@@ -16,7 +16,3 @@ Model
     :undoc-members:
     :show-inheritance:
 
-.. autoclass:: sagemaker.serverless.model.LambdaModel
-    :members:
-    :undoc-members:
-    :show-inheritance:

doc/api/inference/predictors.rst

@@ -7,8 +7,3 @@ Make real-time predictions against SageMaker endpoints with Python objects
     :members:
     :undoc-members:
     :show-inheritance:
-
-.. autoclass:: sagemaker.serverless.predictor.LambdaPredictor
-    :members:
-    :undoc-members:
-    :show-inheritance:

doc/overview.rst

@@ -1063,50 +1063,6 @@ You can also find these notebooks in the **Advanced Functionality** section of t
 For information about using sample notebooks in a SageMaker notebook instance, see `Use Example Notebooks <https://docs.aws.amazon.com/sagemaker/latest/dg/howitworks-nbexamples.html>`__
 in the AWS documentation.
 
-********************
-Serverless Inference
-********************
-
-You can use the SageMaker Python SDK to perform serverless inference on Lambda.
-
-To deploy models to Lambda, you must complete the following prerequisites:
-
-- `Package your model and inference code as a container image. <https://docs.aws.amazon.com/lambda/latest/dg/images-create.html>`_
-- `Create a role that lists Lambda as a trusted entity. <https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html#permissions-executionrole-console>`_
-
-After completing the prerequisites, you can deploy your model to Lambda using
-the `LambdaModel`_ class.
-
-.. code:: python
-
-   from sagemaker.serverless import LambdaModel
-
-   image_uri = "123456789012.dkr.ecr.us-west-2.amazonaws.com/my-lambda-repository:latest"
-   role = "arn:aws:iam::123456789012:role/MyLambdaExecutionRole"
-
-   model = LambdaModel(image_uri=image_uri, role=role)
-   predictor = model.deploy("my-lambda-function", timeout=20, memory_size=4092)
-
-The ``deploy`` method returns a `LambdaPredictor`_  instance. Use the
-`LambdaPredictor`_ ``predict`` method to perform inference on Lambda.
-
-.. code:: python
-
-   url = "https://example.com/cat.jpeg"
-   predictor.predict({"url": url})  # {'class': 'tabby'}
-
-Once you are done performing inference on Lambda, free the `LambdaModel`_ and
-`LambdaPredictor`_ resources using the ``delete_model`` and ``delete_predictor``
-methods.
-
-.. code:: python
-
-   model.delete_model()
-   predictor.delete_predictor()
-
-.. _LambdaModel : https://sagemaker.readthedocs.io/en/stable/api/inference/model.html#sagemaker.serverless.model.LambdaModel
-.. _LambdaPredictor : https://sagemaker.readthedocs.io/en/stable/api/inference/predictors.html#sagemaker.serverless.predictor.LambdaPredictor
-
 ******************
 SageMaker Workflow
 ******************

src/sagemaker/clarify.py

@@ -305,12 +305,13 @@ class SHAPConfig(ExplainabilityConfig):
 
     def __init__(
         self,
-        baseline,
-        num_samples,
-        agg_method,
+        baseline=None,
+        num_samples=None,
+        agg_method=None,
         use_logit=False,
         save_local_shap_values=True,
         seed=None,
+        num_clusters=None,
     ):
         """Initializes config for SHAP.
 
@@ -320,34 +321,49 @@ def __init__(
                 be the same as the dataset format. Each row should contain only the feature
                 columns/values and omit the label column/values. If None a baseline will be
                 calculated automatically by using K-means or K-prototypes in the input dataset.
-            num_samples (int): Number of samples to be used in the Kernel SHAP algorithm.
+            num_samples (None or int): Number of samples to be used in the Kernel SHAP algorithm.
                 This number determines the size of the generated synthetic dataset to compute the
-                SHAP values.
-            agg_method (str): Aggregation method for global SHAP values. Valid values are
+                SHAP values. If not provided then Clarify job will choose a proper value according
+                to the count of features.
+            agg_method (None or str): Aggregation method for global SHAP values. Valid values are
                 "mean_abs" (mean of absolute SHAP values for all instances),
                 "median" (median of SHAP values for all instances) and
                 "mean_sq" (mean of squared SHAP values for all instances).
+                If not provided then Clarify job uses method "mean_abs"
             use_logit (bool): Indicator of whether the logit function is to be applied to the model
                 predictions. Default is False. If "use_logit" is true then the SHAP values will
                 have log-odds units.
             save_local_shap_values (bool): Indicator of whether to save the local SHAP values
                 in the output location. Default is True.
             seed (int): seed value to get deterministic SHAP values. Default is None.
+            num_clusters (None or int): If a baseline is not provided, Clarify automatically
+                computes a baseline dataset via a clustering algorithm (K-means/K-prototypes).
+                num_clusters is a parameter for this algorithm. num_clusters will be the resulting
+                size of the baseline dataset. If not provided, Clarify job will use a default value.
         """
-        if agg_method not in ["mean_abs", "median", "mean_sq"]:
+        if agg_method is not None and agg_method not in ["mean_abs", "median", "mean_sq"]:
             raise ValueError(
                 f"Invalid agg_method {agg_method}." f" Please choose mean_abs, median, or mean_sq."
             )
-
+        if num_clusters is not None and baseline is not None:
+            raise ValueError(
+                "Baseline and num_clusters cannot be provided together. "
+                "Please specify one of the two."
+            )
         self.shap_config = {
-            "baseline": baseline,
-            "num_samples": num_samples,
-            "agg_method": agg_method,
             "use_logit": use_logit,
             "save_local_shap_values": save_local_shap_values,
         }
+        if baseline is not None:
+            self.shap_config["baseline"] = baseline
+        if num_samples is not None:
+            self.shap_config["num_samples"] = num_samples
+        if agg_method is not None:
+            self.shap_config["agg_method"] = agg_method
         if seed is not None:
             self.shap_config["seed"] = seed
+        if num_clusters is not None:
+            self.shap_config["num_clusters"] = num_clusters
 
     def get_explainability_config(self):
         """Returns config."""

src/sagemaker/deprecations.py

@@ -21,24 +21,27 @@
 V2_URL = "https://sagemaker.readthedocs.io/en/stable/v2.html"
 
 
-def _warn(msg):
+def _warn(msg, sdk_version=None):
     """Generic warning raiser referencing V2
 
     Args:
         phrase: The phrase to include in the warning.
+        sdk_version: the sdk version of removal of support.
     """
-    full_msg = f"{msg} in sagemaker>=2.\nSee: {V2_URL} for details."
+    _sdk_version = sdk_version if sdk_version is not None else "2"
+    full_msg = f"{msg} in sagemaker>={_sdk_version}.\nSee: {V2_URL} for details."
     warnings.warn(full_msg, DeprecationWarning, stacklevel=2)
     logger.warning(full_msg)
 
 
-def removed_warning(phrase):
+def removed_warning(phrase, sdk_version=None):
     """Raise a warning for a no-op in sagemaker>=2
 
     Args:
         phrase: the prefix phrase of the warning message.
+        sdk_version: the sdk version of removal of support.
     """
-    _warn(f"{phrase} is a no-op")
+    _warn(f"{phrase} is a no-op", sdk_version)
 
 
 def renamed_warning(phrase):
@@ -146,26 +149,32 @@ def func(*args, **kwargs):  # pylint: disable=W0613
     return func
 
 
-def deprecated(obj):
+def deprecated(sdk_version=None):
     """Decorator for raising deprecated warning for a feature in sagemaker>=2
 
+    Args:
+        sdk_version (str): the sdk version of removal of support.
+
     Usage:
-        @deprecated
+        @deprecated()
         def sample_function():
             print("xxxx....")
 
-        @deprecated
+        @deprecated(sdk_version="2.66")
         class SampleClass():
             def __init__(self):
                 print("xxxx....")
 
     """
 
-    def wrapper(*args, **kwargs):
-        removed_warning(obj.__name__)
-        return obj(*args, **kwargs)
+    def deprecate(obj):
+        def wrapper(*args, **kwargs):
+            removed_warning(obj.__name__, sdk_version)
+            return obj(*args, **kwargs)
+
+        return wrapper
 
-    return wrapper
+    return deprecate
 
 
 def deprecated_function(func, name):

src/sagemaker/estimator.py

@@ -11,63 +11,54 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 """Placeholder docstring"""
-from __future__ import print_function, absolute_import
+from __future__ import absolute_import, print_function
 
 import json
 import logging
 import os
 import uuid
+from abc import ABCMeta, abstractmethod
 
-from abc import ABCMeta
-from abc import abstractmethod
-
-from six import with_metaclass
-from six import string_types
+from six import string_types, with_metaclass
 from six.moves.urllib.parse import urlparse
+
 import sagemaker
-from sagemaker import git_utils, image_uris
+from sagemaker import git_utils, image_uris, vpc_utils
 from sagemaker.analytics import TrainingJobAnalytics
-from sagemaker.debugger import TensorBoardOutputConfig  # noqa: F401 # pylint: disable=unused-import
-from sagemaker.debugger import (
+from sagemaker.debugger import (  # noqa: F401 # pylint: disable=unused-import
     DEBUGGER_FLAG,
     DebuggerHookConfig,
     FrameworkProfile,
-    get_default_profiler_rule,
-    get_rule_container_image_uri,
     ProfilerConfig,
     ProfilerRule,
     Rule,
+    TensorBoardOutputConfig,
+    get_default_profiler_rule,
+    get_rule_container_image_uri,
 )
-from sagemaker.deprecations import (
-    removed_kwargs,
-    removed_function,
-    renamed_kwargs,
-)
-from sagemaker.s3 import S3Uploader, parse_s3_url
-
+from sagemaker.deprecations import removed_function, removed_kwargs, renamed_kwargs
 from sagemaker.fw_utils import (
-    tar_and_upload_dir,
     UploadedCode,
-    validate_source_dir,
     _region_supports_debugger,
     _region_supports_profiler,
     get_mp_parameters,
+    tar_and_upload_dir,
+    validate_source_dir,
 )
-from sagemaker.workflow.properties import Properties
-from sagemaker.workflow.parameters import Parameter
-from sagemaker.workflow.entities import Expression
 from sagemaker.inputs import TrainingInput
 from sagemaker.job import _Job
 from sagemaker.local import LocalSession
-from sagemaker.model import Model, NEO_ALLOWED_FRAMEWORKS
 from sagemaker.model import (
-    SCRIPT_PARAM_NAME,
-    DIR_PARAM_NAME,
     CONTAINER_LOG_LEVEL_PARAM_NAME,
+    DIR_PARAM_NAME,
     JOB_NAME_PARAM_NAME,
+    NEO_ALLOWED_FRAMEWORKS,
     SAGEMAKER_REGION_PARAM_NAME,
+    SCRIPT_PARAM_NAME,
+    Model,
 )
 from sagemaker.predictor import Predictor
+from sagemaker.s3 import S3Uploader, parse_s3_url
 from sagemaker.session import Session
 from sagemaker.transformer import Transformer
 from sagemaker.utils import (
@@ -77,7 +68,9 @@
     get_config_value,
     name_from_base,
 )
-from sagemaker import vpc_utils
+from sagemaker.workflow.entities import Expression
+from sagemaker.workflow.parameters import Parameter
+from sagemaker.workflow.properties import Properties
 
 logger = logging.getLogger(__name__)
 
@@ -150,11 +143,14 @@ def __init__(
                 60 * 60). After this amount of time Amazon SageMaker terminates
                 the job regardless of its current status.
             input_mode (str): The input mode that the algorithm supports
-                (default: 'File'). Valid modes: 'File' - Amazon SageMaker copies
-                the training dataset from the S3 location to a local directory.
+                (default: 'File'). Valid modes:
+                'File' - Amazon SageMaker copiesthe training dataset from the
+                S3 location to a local directory.
                 'Pipe' - Amazon SageMaker streams data directly from S3 to the
-                container via a Unix-named pipe. This argument can be overriden
-                on a per-channel basis using
+                container via a Unix-named pipe.
+                'FastFile' - Amazon SageMaker streams data from S3 on demand instead of
+                downloading the entire dataset before training begins. This argument can
+                be overriden on a per-channel basis using
                 ``sagemaker.inputs.TrainingInput.input_mode``.
             output_path (str): S3 location for saving the training result (model
                 artifacts and output files). If not specified, results are