Merge branch 'master' into feature-3563-add-lambda-func-parameters

# Conflicts:
#	src/sagemaker/lambda_helper.py

Author: dipanjank

CHANGELOG.md

@@ -1,5 +1,123 @@
 # Changelog
 
+## v2.135.0 (2023-02-23)
+
+### Features
+
+ * Add DLC accounts for MEL Region
+ * allow use of short lived creds for local container
+
+### Bug Fixes and Other Changes
+
+ * update lambda function when function arn is provided
+
+## v2.134.1 (2023-02-22)
+
+### Bug Fixes and Other Changes
+
+ * local mode deletion of temp files on job end
+ * Cron expression resetting on update monitor
+ * added support to update arguments in create_monitoring_schedule
+
+## v2.134.0 (2023-02-22)
+
+### Features
+
+ * Add python 3.9 and spark 3.2 support for spark processor
+ * Adding support for Multi Worker Mirrored Strategy in TF estimator
+
+### Bug Fixes and Other Changes
+
+ * tag permission issue - remove describe before create
+
+## v2.133.0 (2023-02-18)
+
+### Features
+
+ * feature store with_feature_group functionality changes
+ * Adding support for SageMaker Training Compiler PyTorch 1.13
+ * support of the intelligent stopping in the tuner
+ * AutoGluon 0.6.2 image_uris update
+ * Support for flexible instance types in the HPO
+ * Add business details and hyper parameters fields and update test_model_card.py
+
+### Bug Fixes and Other Changes
+
+ * disable the tuner test
+ * Skip test_run_from_transform_job integ test to unblock python-sdk code pipeline
+ * Revert "feature: feature store with_feature_group functionality changes"
+ * advanced inference recommendation jobs parameters check
+ * make model_config optional when predicted labels are provided for bias detection
+
+## v2.132.0 (2023-02-07)
+
+### Features
+
+ * support cluster lifecycle management for Sagemaker EMR step
+ * Inference recommendation id deployment support
+
+## v2.131.1 (2023-02-03)
+
+### Bug Fixes and Other Changes
+
+ * test dub gpu integs with p3
+ * fix(experiments/run.py): Stop duplication of RUN_TC_TAG on Consecutive Experiment Runs
+ * Enable load_run without name args in Transform env
+ * Remove confusing log line emitted during feature group ingestion
+ * Enable Experiment integ test on beta clients
+ * Make test_processor_with_role_as_pipeline_parameter more concrete
+
+### Documentation Changes
+
+ * add security note for the estimator hyperparameter arg
+ * SageMaker distributed - model parallism library release note
+ * Add a deprecation note for DetailedProfileConfig
+
+## v2.131.0 (2023-01-31)
+
+### Features
+
+ * Display file diff on black-check
+ * Support for environment variables in the HPO
+ * Support role as PipelineParameter in Processor class
+ * Add TrainingImageConfig support for SageMaker training jobs
+
+### Bug Fixes and Other Changes
+
+ * use FeatureGroup's Session in nonconcurrency ingestion
+ * Update feature_group.py ingest() description
+ * Do not use print function. User logger instead
+ * Add batch_get_record and search API for FeatureStore
+ * hashing problem for framework processors with identical source dirs
+
+## v2.130.0 (2023-01-26)
+
+### Features
+
+ * Add PyTorch 1.13.1 to SDK
+ * Adding image_uri config for DJL containers
+ * Support specifying env-vars when creating model from model package
+ * local download dir for Model and Estimator classes
+
+### Bug Fixes and Other Changes
+
+ * increase creation time slack minutes
+ * Enable load_run auto pass in experiment config
+ * Add us-isob-east-1 accounts and configs
+ * Clean up Pipeline unit tests
+
+## v2.129.0 (2023-01-19)
+
+### Features
+
+ * add p2 deprecation for PT>=1.13
+ * TF2.11 Update to PySDK
+
+### Bug Fixes and Other Changes
+
+ * Improve Pipeline integ tests and fix resource leak
+ * Update TF version to 2.8.4
+
 ## v2.128.0 (2023-01-10)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.128.1.dev0
+2.135.1.dev0

doc/api/governance/model_card.rst

@@ -41,3 +41,9 @@ see `Amazon SageMaker Model Cards <https://docs.aws.amazon.com/sagemaker/latest/
 
 .. autoclass:: TrainingJobDetails
     :show-inheritance:
+
+.. autoclass:: BusinessDetails
+    :show-inheritance:
+
+.. autoclass:: HyperParameter
+    :show-inheritance:

doc/api/training/sdp_versions/latest.rst

@@ -26,8 +26,8 @@ depending on the version of the library you use.
    <https://docs.aws.amazon.com/sagemaker/latest/dg/data-parallel-use-api.html#data-parallel-use-python-skd-api>`_
    for more information.
 
-Version 1.4.0, 1.4.1, 1.5.0, 1.6.0 (Latest)
-===========================================
+For versions between 1.4.0 and 1.7.0 (Latest)
+=============================================
 
 .. toctree::
    :maxdepth: 1

doc/api/training/smd_data_parallel_release_notes/smd_data_parallel_change_log.rst

@@ -7,9 +7,40 @@ Release Notes
 New features, bug fixes, and improvements are regularly made to the SageMaker
 distributed data parallel library.
 
-SageMaker Distributed Data Parallel 1.6.0 Release Notes
+SageMaker Distributed Data Parallel 1.7.0 Release Notes
 =======================================================
 
+*Date: Feb. 10. 2023*
+
+**Currency Updates**
+
+* Added support for PyTorch 1.13.1.
+
+**Migration to AWS Deep Learning Containers**
+
+This version passed benchmark testing and is migrated to the following AWS Deep Learning Containers (DLC):
+
+- PyTorch 1.13.1 DLC
+
+  .. code::
+
+    763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.13.1-gpu-py39-cu117-ubuntu20.04-sagemaker
+
+Binary file of this version of the library for custom container users:
+
+  .. code::
+
+    https://smdataparallel.s3.amazonaws.com/binary/pytorch/1.13.1/cu117/2023-01-09/smdistributed_dataparallel-1.7.0-cp39-cp39-linux_x86_64.whl
+
+
+----
+
+Release History
+===============
+
+SageMaker Distributed Data Parallel 1.6.0 Release Notes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 *Date: Dec. 15. 2022*
 
 **New Features**
@@ -44,11 +75,6 @@ Binary file of this version of the library for `custom container
     https://smdataparallel.s3.amazonaws.com/binary/pytorch/1.12.1/cu113/2022-12-05/smdistributed_dataparallel-1.6.0-cp38-cp38-linux_x86_64.whl
 
 
-----
-
-Release History
-===============
-
 SageMaker Distributed Data Parallel 1.5.0 Release Notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/api/training/smd_model_parallel_release_notes/smd_model_parallel_change_log.rst

@@ -6,9 +6,47 @@ New features, bug fixes, and improvements are regularly made to the SageMaker
 distributed model parallel library.
 
 
-SageMaker Distributed Model Parallel 1.13.0 Release Notes
+SageMaker Distributed Model Parallel 1.14.0 Release Notes
 =========================================================
 
+*Date: Jan. 30. 2023*
+
+**Currency Updates**
+
+* Added support for PyTorch v1.13.1
+
+**Improvements**
+
+* Upgraded the flash-attention (https://github.com/HazyResearch/flash-attention) library to  v0.2.6.post1
+
+**Migration to AWS Deep Learning Containers**
+
+This version passed benchmark testing and is migrated to the following AWS Deep Learning Containers (DLC):
+
+- SageMaker training container for PyTorch v1.13.1
+
+  .. code::
+
+    763104351884.dkr.ecr.<region>.amazonaws.com/pytorch-training:1.13.1-gpu-py39-cu117-ubuntu20.04-sagemaker
+
+
+Binary file of this version of the library for `custom container
+<https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-sm-sdk.html#model-parallel-bring-your-own-container>`_ users:
+
+- For PyTorch 1.13.1
+
+  .. code::
+
+    https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.13.1/build-artifacts/2023-01-19-18-35/smdistributed_modelparallel-1.14.0-cp39-cp39-linux_x86_64.whl
+
+----
+
+Release History
+===============
+
+SageMaker Distributed Model Parallel 1.13.0 Release Notes
+---------------------------------------------------------
+
 *Date: Dec. 15. 2022*
 
 **New Features**
@@ -46,16 +84,12 @@ This version passed benchmark testing and is migrated to the following AWS Deep
 Binary file of this version of the library for `custom container
 <https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-sm-sdk.html#model-parallel-bring-your-own-container>`_ users:
 
-- For PyTorch 1.12.0
+- For PyTorch 1.12.1
 
   .. code::
 
     https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.12.1/build-artifacts/2022-12-08-21-34/smdistributed_modelparallel-1.13.0-cp38-cp38-linux_x86_64.whl
 
-----
-
-Release History
-===============
 
 SageMaker Distributed Model Parallel 1.11.0 Release Notes
 ---------------------------------------------------------
@@ -92,7 +126,7 @@ Binary file of this version of the library for `custom container
 
   .. code::
 
-    https://sagemaker-distribu
+    https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.12.0/build-artifacts/2022-08-12-16-58/smdistributed_modelparallel-1.11.0-cp38-cp38-linux_x86_64.whl
 
 SageMaker Distributed Model Parallel 1.10.1 Release Notes
 ---------------------------------------------------------

doc/api/training/smp_versions/latest.rst

@@ -10,8 +10,8 @@ depending on which version of the library you need to use.
 To use the library, reference the
 **Common API** documentation alongside the framework specific API documentation.
 
-Version 1.11.0, 1.13.0 (Latest)
-===============================
+Version 1.11.0, 1.13.0, 1.14.0 (Latest)
+=======================================
 
 To use the library, reference the Common API documentation alongside the framework specific API documentation.
 

doc/overview.rst

@@ -1578,6 +1578,7 @@ A few important notes:
 - If you are using S3 data as input, it is pulled from S3 to your local environment. Ensure you have sufficient space to store the data locally.
 - If you run into problems it often due to different Docker containers conflicting. Killing these containers and re-running often solves your problems.
 - Local Mode requires Docker Compose and `nvidia-docker2 <https://github.com/NVIDIA/nvidia-docker>`__ for ``local_gpu``.
+- Set ``USE_SHORT_LIVED_CREDENTIALS=1`` if running on EC2 and you would like to use the session credentials instead of EC2 Metadata Service credentials.
 
 .. warning::
 

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -169,4 +169,8 @@ Steps
 
 .. autoclass:: sagemaker.workflow.fail_step.FailStep
 
+.. autoclass:: sagemaker.workflow.emr_step.EMRStepConfig
+
+.. autoclass:: sagemaker.workflow.emr_step.EMRStep
+
 .. autoclass:: sagemaker.workflow.automl_step.AutoMLStep

src/sagemaker/clarify.py

@@ -1423,8 +1423,8 @@ def run_post_training_bias(
         self,
         data_config: DataConfig,
         data_bias_config: BiasConfig,
-        model_config: ModelConfig,
-        model_predicted_label_config: ModelPredictedLabelConfig,
+        model_config: Optional[ModelConfig] = None,
+        model_predicted_label_config: Optional[ModelPredictedLabelConfig] = None,
         methods: Union[str, List[str]] = "all",
         wait: bool = True,
         logs: bool = True,
@@ -1444,7 +1444,8 @@ def run_post_training_bias(
             data_config (:class:`~sagemaker.clarify.DataConfig`): Config of the input/output data.
             data_bias_config (:class:`~sagemaker.clarify.BiasConfig`): Config of sensitive groups.
             model_config (:class:`~sagemaker.clarify.ModelConfig`): Config of the model and its
-                endpoint to be created.
+                endpoint to be created. This is required unless``predicted_label_dataset_uri`` or
+                ``predicted_label`` is provided in ``data_config``.
             model_predicted_label_config (:class:`~sagemaker.clarify.ModelPredictedLabelConfig`):
                 Config of how to extract the predicted label from the model output.
             methods (str or list[str]): Selector of a subset of potential metrics:
@@ -1508,7 +1509,7 @@ def run_bias(
         self,
         data_config: DataConfig,
         bias_config: BiasConfig,
-        model_config: ModelConfig,
+        model_config: Optional[ModelConfig] = None,
         model_predicted_label_config: Optional[ModelPredictedLabelConfig] = None,
         pre_training_methods: Union[str, List[str]] = "all",
         post_training_methods: Union[str, List[str]] = "all",
@@ -1529,7 +1530,8 @@ def run_bias(
             data_config (:class:`~sagemaker.clarify.DataConfig`): Config of the input/output data.
             bias_config (:class:`~sagemaker.clarify.BiasConfig`): Config of sensitive groups.
             model_config (:class:`~sagemaker.clarify.ModelConfig`): Config of the model and its
-                endpoint to be created.
+                endpoint to be created. This is required unless``predicted_label_dataset_uri`` or
+                ``predicted_label`` is provided in ``data_config``.
             model_predicted_label_config (:class:`~sagemaker.clarify.ModelPredictedLabelConfig`):
                 Config of how to extract the predicted label from the model output.
             pre_training_methods (str or list[str]): Selector of a subset of potential metrics:
@@ -1930,16 +1932,30 @@ def _add_predictor(
     ):
         """Extends analysis config with predictor."""
         analysis_config = {**analysis_config}
-        analysis_config["predictor"] = model_config.get_predictor_config()
+        if isinstance(model_config, ModelConfig):
+            analysis_config["predictor"] = model_config.get_predictor_config()
+        else:
+            if "shap" in analysis_config["methods"] or "pdp" in analysis_config["methods"]:
+                raise ValueError(
+                    "model_config must be provided when explainability methods are selected."
+                )
+            if (
+                "predicted_label_dataset_uri" not in analysis_config
+                and "predicted_label" not in analysis_config
+            ):
+                raise ValueError(
+                    "model_config must be provided when `predicted_label_dataset_uri` or "
+                    "`predicted_label` are not provided in data_config."
+                )
         if isinstance(model_predicted_label_config, ModelPredictedLabelConfig):
             (
                 probability_threshold,
                 predictor_config,
             ) = model_predicted_label_config.get_predictor_config()
-            if predictor_config:
+            if predictor_config and "predictor" in analysis_config:
                 analysis_config["predictor"].update(predictor_config)
             _set(probability_threshold, "probability_threshold", analysis_config)
-        else:
+        elif "predictor" in analysis_config:
             _set(model_predicted_label_config, "label", analysis_config["predictor"])
         return analysis_config
 

src/sagemaker/debugger/framework_profile.py

@@ -143,6 +143,12 @@ def __init__(
                 profiling. Configure it using the
                 :class:`~sagemaker.debugger.metrics_config.DetailedProfilingConfig` class.
                 Pass ``DetailedProfilingConfig()`` to use the default configuration.
+
+                .. warning::
+                    This detailed framework profiling feature discontinues support for TensorFlow v2.11
+                    and later. To use the detailed profiling feature, use previous versions of
+                    TensorFlow between v2.3.1 and v2.10.0.
+
             dataloader_profiling_config (DataloaderProfilingConfig): The configuration for
                 dataloader metrics profiling. Configure it using the
                 :class:`~sagemaker.debugger.metrics_config.DataloaderProfilingConfig` class.