Merge branch 'master' into ptddp-launcher

Author: vishwakaria

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## v2.101.0 (2022-07-27)
+
+### Features
+
+ * Algorithms region launch on CGK
+ * enhance-bucket-override-support
+ * infer framework and version
+ * support clarify bias detection when facets not included
+ * Add CGK region to frameworks by DLC
+
+### Bug Fixes and Other Changes
+
+ * Make repack step output path align with model repack path
+ * Support parameterized source code input for TrainingStep
+
+### Documentation Changes
+
+ * heterogeneous cluster api doc fix
+ * smdmp v1.10 release note
+
 ## v2.100.0 (2022-07-18)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.100.1.dev0
+2.101.1.dev0

doc/api/training/smd_model_parallel_release_notes/smd_model_parallel_change_log.rst

@@ -5,14 +5,31 @@ Release Notes
 New features, bug fixes, and improvements are regularly made to the SageMaker
 distributed model parallel library.
 
-SageMaker Distributed Model Parallel 1.9.0 Release Notes
-========================================================
+SageMaker Distributed Model Parallel 1.10.0 Release Notes
+=========================================================
 
-*Date: May. 3. 2022*
+*Date: July. 19. 2022*
 
-**Currency Updates**
+**New Features**
 
-* Added support for PyTorch 1.11.0
+The following new features are added for PyTorch.
+
+* Added support for FP16 training by implementing smdistributed.modelparallel
+  modification of Apex FP16_Module and FP16_Optimizer. To learn more, see
+  `FP16 Training with Model Parallelism
+  <https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-extended-features-pytorch-fp16.html>`_.
+* New checkpoint APIs for CPU memory usage optimization. To learn more, see
+  `Checkpointing Distributed Models and Optimizer States
+  <https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-extended-features-pytorch-checkpoint.html>`_.
+
+**Improvements**
+
+* The SageMaker distributed model parallel library manages and optimizes CPU
+  memory by garbage-collecting non-local parameters in general and during checkpointing.
+* Changes in the `GPT-2 translate functions
+  <https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-extended-features-pytorch-hugging-face.html>`_
+  (``smdistributed.modelparallel.torch.nn.huggingface.gpt2``)
+  to save memory by not maintaining two copies of weights at the same time.
 
 **Migration to AWS Deep Learning Containers**
 
@@ -28,7 +45,7 @@ Binary file of this version of the library for custom container users:
 
   .. code::
 
-    https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.11.0/build-artifacts/2022-04-20-17-05/smdistributed_modelparallel-1.9.0-cp38-cp38-linux_x86_64.whl
+    https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.11.0/build-artifacts/2022-07-11-19-23/smdistributed_modelparallel-1.10.0-cp38-cp38-linux_x86_64.whl
 
 
 
@@ -37,6 +54,33 @@ Binary file of this version of the library for custom container users:
 Release History
 ===============
 
+SageMaker Distributed Model Parallel 1.9.0 Release Notes
+--------------------------------------------------------
+
+*Date: May. 3. 2022*
+
+**Currency Updates**
+
+* Added support for PyTorch 1.11.0
+
+**Migration to AWS Deep Learning Containers**
+
+This version passed benchmark testing and is migrated to the following AWS Deep Learning Containers (DLC):
+
+- PyTorch 1.11.0 DLC
+
+  .. code::
+
+    763104351884.dkr.ecr.<region>.amazonaws.com/pytorch-training:1.11.0-gpu-py38-cu113-ubuntu20.04-sagemaker
+
+Binary file of this version of the library for custom container users:
+
+  .. code::
+
+    https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.11.0/build-artifacts/2022-04-20-17-05/smdistributed_modelparallel-1.9.0-cp38-cp38-linux_x86_64.whl
+
+
+
 SageMaker Distributed Model Parallel 1.8.1 Release Notes
 --------------------------------------------------------
 

doc/overview.rst

@@ -1713,11 +1713,15 @@ in the AWS documentation.
 SageMaker Workflow
 ******************
 
-You can use Apache Airflow to author, schedule and monitor SageMaker workflow.
+You can use the following machine learning frameworks to author, schedule and monitor SageMaker workflow.
 
-For more information, see `SageMaker Workflow in Apache Airflow`_.
+.. toctree::
+    :maxdepth: 2
 
-.. _SageMaker Workflow in Apache Airflow: https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/workflow/README.rst
+    workflows/airflow/index
+    workflows/step_functions/index
+    workflows/pipelines/index
+    workflows/lineage/index
 
 ************************************
 SageMaker Model Building Pipeline

src/sagemaker/amazon/factorization_machines.py

@@ -13,6 +13,8 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
+from typing import Union, Optional
+
 from sagemaker import image_uris
 from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase
 from sagemaker.amazon.common import RecordSerializer, RecordDeserializer
@@ -21,7 +23,9 @@
 from sagemaker.predictor import Predictor
 from sagemaker.model import Model
 from sagemaker.session import Session
+from sagemaker.utils import pop_out_unused_kwarg
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
+from sagemaker.workflow.entities import PipelineVariable
 
 
 class FactorizationMachines(AmazonAlgorithmEstimatorBase):
@@ -319,7 +323,13 @@ class FactorizationMachinesModel(Model):
     returns :class:`FactorizationMachinesPredictor`.
     """
 
-    def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
+    def __init__(
+        self,
+        model_data: Union[str, PipelineVariable],
+        role: str,
+        sagemaker_session: Optional[Session] = None,
+        **kwargs
+    ):
         """Initialization for FactorizationMachinesModel class.
 
         Args:
@@ -343,6 +353,8 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
             sagemaker_session.boto_region_name,
             version=FactorizationMachines.repo_version,
         )
+        pop_out_unused_kwarg("predictor_cls", kwargs, FactorizationMachinesPredictor.__name__)
+        pop_out_unused_kwarg("image_uri", kwargs, image_uri)
         super(FactorizationMachinesModel, self).__init__(
             image_uri,
             model_data,

src/sagemaker/amazon/hyperparameter.py

@@ -15,6 +15,8 @@
 
 import json
 
+from sagemaker.workflow import is_pipeline_variable
+
 
 class Hyperparameter(object):
     """An algorithm hyperparameter with optional validation.
@@ -98,8 +100,14 @@ def serialize_all(obj):
         """
         if "_hyperparameters" not in dir(obj):
             return {}
-        return {
-            k: json.dumps(v) if isinstance(v, list) else str(v)
-            for k, v in obj._hyperparameters.items()
-            if v is not None
-        }
+        hps = {}
+        for k, v in obj._hyperparameters.items():
+            if v is not None:
+                if isinstance(v, list):
+                    v = json.dumps(v)
+                elif is_pipeline_variable(v):
+                    v = v.to_string()
+                else:
+                    v = str(v)
+                hps[k] = v
+        return hps

src/sagemaker/amazon/ipinsights.py

@@ -13,6 +13,8 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
+from typing import Union, Optional
+
 from sagemaker import image_uris
 from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase
 from sagemaker.amazon.hyperparameter import Hyperparameter as hp  # noqa
@@ -22,7 +24,9 @@
 from sagemaker.model import Model
 from sagemaker.serializers import CSVSerializer
 from sagemaker.session import Session
+from sagemaker.utils import pop_out_unused_kwarg
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
+from sagemaker.workflow.entities import PipelineVariable
 
 
 class IPInsights(AmazonAlgorithmEstimatorBase):
@@ -222,7 +226,13 @@ class IPInsightsModel(Model):
     Predictor that calculates anomaly scores for data points.
     """
 
-    def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
+    def __init__(
+        self,
+        model_data: Union[str, PipelineVariable],
+        role: str,
+        sagemaker_session: Optional[Session] = None,
+        **kwargs
+    ):
         """Creates object to get insights on S3 model data.
 
         Args:
@@ -246,6 +256,8 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
             sagemaker_session.boto_region_name,
             version=IPInsights.repo_version,
         )
+        pop_out_unused_kwarg("predictor_cls", kwargs, IPInsightsPredictor.__name__)
+        pop_out_unused_kwarg("image_uri", kwargs, image_uri)
         super(IPInsightsModel, self).__init__(
             image_uri,
             model_data,

src/sagemaker/amazon/kmeans.py

@@ -13,6 +13,8 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
+from typing import Union, Optional
+
 from sagemaker import image_uris
 from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase
 from sagemaker.amazon.common import RecordSerializer, RecordDeserializer
@@ -21,7 +23,9 @@
 from sagemaker.predictor import Predictor
 from sagemaker.model import Model
 from sagemaker.session import Session
+from sagemaker.utils import pop_out_unused_kwarg
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
+from sagemaker.workflow.entities import PipelineVariable
 
 
 class KMeans(AmazonAlgorithmEstimatorBase):
@@ -246,7 +250,13 @@ class KMeansModel(Model):
     Predictor to performs k-means cluster assignment.
     """
 
-    def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
+    def __init__(
+        self,
+        model_data: Union[str, PipelineVariable],
+        role: str,
+        sagemaker_session: Optional[Session] = None,
+        **kwargs
+    ):
         """Initialization for KMeansModel class.
 
         Args:
@@ -270,6 +280,8 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
             sagemaker_session.boto_region_name,
             version=KMeans.repo_version,
         )
+        pop_out_unused_kwarg("predictor_cls", kwargs, KMeansPredictor.__name__)
+        pop_out_unused_kwarg("image_uri", kwargs, image_uri)
         super(KMeansModel, self).__init__(
             image_uri,
             model_data,

src/sagemaker/amazon/knn.py

@@ -13,6 +13,8 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
+from typing import Union, Optional
+
 from sagemaker import image_uris
 from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase
 from sagemaker.amazon.common import RecordSerializer, RecordDeserializer
@@ -21,7 +23,9 @@
 from sagemaker.predictor import Predictor
 from sagemaker.model import Model
 from sagemaker.session import Session
+from sagemaker.utils import pop_out_unused_kwarg
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
+from sagemaker.workflow.entities import PipelineVariable
 
 
 class KNN(AmazonAlgorithmEstimatorBase):
@@ -238,7 +242,13 @@ class KNNModel(Model):
     and returns :class:`KNNPredictor`.
     """
 
-    def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
+    def __init__(
+        self,
+        model_data: Union[str, PipelineVariable],
+        role: str,
+        sagemaker_session: Optional[Session] = None,
+        **kwargs
+    ):
         """Function to initialize KNNModel.
 
         Args:
@@ -262,6 +272,8 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
             sagemaker_session.boto_region_name,
             version=KNN.repo_version,
         )
+        pop_out_unused_kwarg("predictor_cls", kwargs, KNNPredictor.__name__)
+        pop_out_unused_kwarg("image_uri", kwargs, image_uri)
         super(KNNModel, self).__init__(
             image_uri,
             model_data,

src/sagemaker/amazon/lda.py

@@ -13,6 +13,8 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
+from typing import Union, Optional
+
 from sagemaker import image_uris
 from sagemaker.amazon.amazon_estimator import AmazonAlgorithmEstimatorBase
 from sagemaker.amazon.common import RecordSerializer, RecordDeserializer
@@ -21,7 +23,9 @@
 from sagemaker.predictor import Predictor
 from sagemaker.model import Model
 from sagemaker.session import Session
+from sagemaker.utils import pop_out_unused_kwarg
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
+from sagemaker.workflow.entities import PipelineVariable
 
 
 class LDA(AmazonAlgorithmEstimatorBase):
@@ -220,7 +224,13 @@ class LDAModel(Model):
     Predictor that transforms vectors to a lower-dimensional representation.
     """
 
-    def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
+    def __init__(
+        self,
+        model_data: Union[str, PipelineVariable],
+        role: str,
+        sagemaker_session: Optional[Session] = None,
+        **kwargs
+    ):
         """Initialization for LDAModel class.
 
         Args:
@@ -244,6 +254,8 @@ def __init__(self, model_data, role, sagemaker_session=None, **kwargs):
             sagemaker_session.boto_region_name,
             version=LDA.repo_version,
         )
+        pop_out_unused_kwarg("predictor_cls", kwargs, LDAPredictor.__name__)
+        pop_out_unused_kwarg("image_uri", kwargs, image_uri)
         super(LDAModel, self).__init__(
             image_uri,
             model_data,