Merge branch 'master' into feat/enhance-bucket-override-support

Author: evakravi

doc/api/training/smd_model_parallel_release_notes/smd_model_parallel_change_log.rst

@@ -5,14 +5,31 @@ Release Notes
 New features, bug fixes, and improvements are regularly made to the SageMaker
 distributed model parallel library.
 
-SageMaker Distributed Model Parallel 1.9.0 Release Notes
-========================================================
+SageMaker Distributed Model Parallel 1.10.0 Release Notes
+=========================================================
 
-*Date: May. 3. 2022*
+*Date: July. 19. 2022*
 
-**Currency Updates**
+**New Features**
 
-* Added support for PyTorch 1.11.0
+The following new features are added for PyTorch.
+
+* Added support for FP16 training by implementing smdistributed.modelparallel
+  modification of Apex FP16_Module and FP16_Optimizer. To learn more, see
+  `FP16 Training with Model Parallelism
+  <https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-extended-features-pytorch-fp16.html>`_.
+* New checkpoint APIs for CPU memory usage optimization. To learn more, see
+  `Checkpointing Distributed Models and Optimizer States
+  <https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-extended-features-pytorch-checkpoint.html>`_.
+
+**Improvements**
+
+* The SageMaker distributed model parallel library manages and optimizes CPU
+  memory by garbage-collecting non-local parameters in general and during checkpointing.
+* Changes in the `GPT-2 translate functions
+  <https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-extended-features-pytorch-hugging-face.html>`_
+  (``smdistributed.modelparallel.torch.nn.huggingface.gpt2``)
+  to save memory by not maintaining two copies of weights at the same time.
 
 **Migration to AWS Deep Learning Containers**
 
@@ -28,7 +45,7 @@ Binary file of this version of the library for custom container users:
 
   .. code::
 
-    https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.11.0/build-artifacts/2022-04-20-17-05/smdistributed_modelparallel-1.9.0-cp38-cp38-linux_x86_64.whl
+    https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.11.0/build-artifacts/2022-07-11-19-23/smdistributed_modelparallel-1.10.0-cp38-cp38-linux_x86_64.whl
 
 
 
@@ -37,6 +54,33 @@ Binary file of this version of the library for custom container users:
 Release History
 ===============
 
+SageMaker Distributed Model Parallel 1.9.0 Release Notes
+--------------------------------------------------------
+
+*Date: May. 3. 2022*
+
+**Currency Updates**
+
+* Added support for PyTorch 1.11.0
+
+**Migration to AWS Deep Learning Containers**
+
+This version passed benchmark testing and is migrated to the following AWS Deep Learning Containers (DLC):
+
+- PyTorch 1.11.0 DLC
+
+  .. code::
+
+    763104351884.dkr.ecr.<region>.amazonaws.com/pytorch-training:1.11.0-gpu-py38-cu113-ubuntu20.04-sagemaker
+
+Binary file of this version of the library for custom container users:
+
+  .. code::
+
+    https://sagemaker-distributed-model-parallel.s3.us-west-2.amazonaws.com/pytorch-1.11.0/build-artifacts/2022-04-20-17-05/smdistributed_modelparallel-1.9.0-cp38-cp38-linux_x86_64.whl
+
+
+
 SageMaker Distributed Model Parallel 1.8.1 Release Notes
 --------------------------------------------------------
 

src/sagemaker/amazon/hyperparameter.py

@@ -15,6 +15,8 @@
 
 import json
 
+from sagemaker.workflow import is_pipeline_variable
+
 
 class Hyperparameter(object):
     """An algorithm hyperparameter with optional validation.
@@ -98,8 +100,14 @@ def serialize_all(obj):
         """
         if "_hyperparameters" not in dir(obj):
             return {}
-        return {
-            k: json.dumps(v) if isinstance(v, list) else str(v)
-            for k, v in obj._hyperparameters.items()
-            if v is not None
-        }
+        hps = {}
+        for k, v in obj._hyperparameters.items():
+            if v is not None:
+                if isinstance(v, list):
+                    v = json.dumps(v)
+                elif is_pipeline_variable(v):
+                    v = v.to_string()
+                else:
+                    v = str(v)
+                hps[k] = v
+        return hps

src/sagemaker/chainer/estimator.py

@@ -14,6 +14,7 @@
 from __future__ import absolute_import
 
 import logging
+from typing import Union, Optional
 
 from sagemaker.estimator import Framework, EstimatorBase
 from sagemaker.fw_utils import (
@@ -25,6 +26,7 @@
 from sagemaker.chainer import defaults
 from sagemaker.chainer.model import ChainerModel
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
+from sagemaker.workflow.entities import PipelineVariable
 
 logger = logging.getLogger("sagemaker")
 
@@ -42,12 +44,12 @@ class Chainer(Framework):
 
     def __init__(
         self,
-        entry_point,
+        entry_point: Union[str, PipelineVariable],
         use_mpi=None,
         num_processes=None,
         process_slots_per_host=None,
         additional_mpi_options=None,
-        source_dir=None,
+        source_dir: Optional[Union[str, PipelineVariable]] = None,
         hyperparameters=None,
         framework_version=None,
         py_version=None,

src/sagemaker/chainer/model.py

@@ -140,6 +140,108 @@ def __init__(
 
         self.model_server_workers = model_server_workers
 
+    def register(
+        self,
+        content_types,
+        response_types,
+        inference_instances,
+        transform_instances,
+        model_package_name=None,
+        model_package_group_name=None,
+        image_uri=None,
+        model_metrics=None,
+        metadata_properties=None,
+        marketplace_cert=False,
+        approval_status=None,
+        description=None,
+        drift_check_baselines=None,
+        customer_metadata_properties=None,
+        domain=None,
+        sample_payload_url=None,
+        task=None,
+        framework=None,
+        framework_version=None,
+        nearest_model_name=None,
+        data_input_configuration=None,
+    ):
+        """Creates a model package for creating SageMaker models or listing on Marketplace.
+
+        Args:
+            content_types (list): The supported MIME types for the input data.
+            response_types (list): The supported MIME types for the output data.
+            inference_instances (list): A list of the instance types that are used to
+                generate inferences in real-time.
+            transform_instances (list): A list of the instance types on which a transformation
+                job can be run or on which an endpoint can be deployed.
+            model_package_name (str): Model Package name, exclusive to `model_package_group_name`,
+                using `model_package_name` makes the Model Package un-versioned (default: None).
+            model_package_group_name (str): Model Package Group name, exclusive to
+                `model_package_name`, using `model_package_group_name` makes the Model Package
+                versioned (default: None).
+            image_uri (str): Inference image uri for the container. Model class' self.image will
+                be used if it is None (default: None).
+            model_metrics (ModelMetrics): ModelMetrics object (default: None).
+            metadata_properties (MetadataProperties): MetadataProperties (default: None).
+            marketplace_cert (bool): A boolean value indicating if the Model Package is certified
+                for AWS Marketplace (default: False).
+            approval_status (str): Model Approval Status, values can be "Approved", "Rejected",
+                or "PendingManualApproval" (default: "PendingManualApproval").
+            description (str): Model Package description (default: None).
+            drift_check_baselines (DriftCheckBaselines): DriftCheckBaselines object (default: None).
+            customer_metadata_properties (dict[str, str]): A dictionary of key-value paired
+                metadata properties (default: None).
+            domain (str): Domain values can be "COMPUTER_VISION", "NATURAL_LANGUAGE_PROCESSING",
+                "MACHINE_LEARNING" (default: None).
+            sample_payload_url (str): The S3 path where the sample payload is stored
+                (default: None).
+            task (str): Task values which are supported by Inference Recommender are "FILL_MASK",
+                "IMAGE_CLASSIFICATION", "OBJECT_DETECTION", "TEXT_GENERATION", "IMAGE_SEGMENTATION",
+                "CLASSIFICATION", "REGRESSION", "OTHER" (default: None).
+            framework (str): Machine learning framework of the model package container image
+                (default: None).
+            framework_version (str): Framework version of the Model Package Container Image
+                (default: None).
+            nearest_model_name (str): Name of a pre-trained machine learning benchmarked by
+                Amazon SageMaker Inference Recommender (default: None).
+            data_input_configuration (str): Input object for the model (default: None).
+
+        Returns:
+            str: A string of SageMaker Model Package ARN.
+        """
+        instance_type = inference_instances[0]
+        self._init_sagemaker_session_if_does_not_exist(instance_type)
+
+        if image_uri:
+            self.image_uri = image_uri
+        if not self.image_uri:
+            self.image_uri = self.serving_image_uri(
+                region_name=self.sagemaker_session.boto_session.region_name,
+                instance_type=instance_type,
+            )
+        return super(ChainerModel, self).register(
+            content_types,
+            response_types,
+            inference_instances,
+            transform_instances,
+            model_package_name,
+            model_package_group_name,
+            image_uri,
+            model_metrics,
+            metadata_properties,
+            marketplace_cert,
+            approval_status,
+            description,
+            drift_check_baselines=drift_check_baselines,
+            customer_metadata_properties=customer_metadata_properties,
+            domain=domain,
+            sample_payload_url=sample_payload_url,
+            task=task,
+            framework=(framework or self._framework_name).upper(),
+            framework_version=framework_version or self.framework_version,
+            nearest_model_name=nearest_model_name,
+            data_input_configuration=data_input_configuration,
+        )
+
     def prepare_container_def(
         self, instance_type=None, accelerator_type=None, serverless_inference_config=None
     ):