Merge branch 'master' into patch-1

Author: TheOnly92

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## v2.149.0 (2023-04-25)
+
+### Features
+
+ * Support TF2.12 SageMaker DLC
+
+### Bug Fixes and Other Changes
+
+ * update the doc for Join function
+ * change s3UploadMode of sagemaker clarify processing output for computer vision jobs.
+
+### Documentation Changes
+
+ * Add Remote Function updates
+
 ## v2.148.0 (2023-04-20)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.148.1.dev0
+2.149.1.dev0

doc/overview.rst

@@ -2021,6 +2021,32 @@ set default values for. For the full schema, see ``sagemaker.config.config_schem
         Tags:
         - Key: 'tag_key'
           Value: 'tag_value'
+      PythonSDK:
+        Modules:
+          RemoteFunction:
+            Dependencies: 'path/to/requirements.txt'
+            EnableInterContainerTrafficEncryption: true
+            EnvironmentVariables: {'EnvVarKey': 'EnvVarValue'}
+            ImageUri: '555555555555.dkr.ecr.us-west-2.amazonaws.com/my-image:latest'
+            IncludeLocalWorkDir: true
+            InstanceType: 'ml.m5.large'
+            JobCondaEnvironment: 'your_conda_env'
+            PreExecutionCommands:
+              - 'command_1'
+              - 'command_2'
+            PreExecutionScript: 'path/to/script.sh'
+            RoleArn: 'arn:aws:iam::555555555555:role/MyRole'
+            S3KmsKeyId: 'yourkmskeyid'
+            S3RootUri: 's3://my-bucket/my-project'
+            VpcConfig:
+              SecurityGroupIds:
+                - 'sg123'
+              Subnets:
+                - 'subnet-1234'
+            Tags:
+              - Key: 'tag_key'
+                Value: 'tag_value'
+            VolumeKmsKeyId: 'yourkmskeyid'
 
 Configuration file locations
 ============================
@@ -2170,8 +2196,8 @@ types support setting defaults with a configuration file.
 -  Tags
 -  Enable inter-container traffic encryption
 
-List of APIs supported
-----------------------
+List of APIs and SDK capabilities supported
+-------------------------------------------
 
 Default values for the supported parameters of these APIs apply to
 all create and update calls for that API. For example, if a supported
@@ -2202,6 +2228,10 @@ configuration file.
 Hyperparameter Tuning Job: Supported indirectly via ``TrainingJob`` API. While this API is not directly supported, it includes the training job definition as a parameter.
 If you provide defaults for this parameter as part of the ``TrainingJob`` API, these defaults are also used for Hyperparameter Tuning Job.
 
+The following goups of SDK capabilities support defaults with a configuration file.
+
+-  Remote Function ``@remote decorator``, ``RemoteExecutor```
+
 Configuration file resolution
 =============================
 
@@ -2413,6 +2443,45 @@ specifically the contents of ``'body': b'{...}`` .
    'query_string': ..., 'method': 'POST', 'headers': {...}, 'body': b'{...}', 'url': 'https://api.sagemaker.us-west-2.amazonaws.com/',
    'context': {...}}
 
+
+************************************************************
+Run Machine Learning code on SageMaker using remote function
+************************************************************
+
+You can integrate your local machine language (ML) code to run in a Amazon SageMaker Training job by wrapping
+your code inside a @remote decorator as shown in the following code example.
+
+.. code-block:: python
+
+    from sagemaker.remote_function import remote
+    import numpy as np
+
+    @remote(instance_type="ml.m5.large")
+    def matrix_multiply(a, b):
+        return np.matmul(a, b)
+
+    a = np.array([[1, 0],
+                 [0, 1]])
+    b = np.array([1, 2])
+
+    assert (matrix_multiply(a, b) == np.array([1,2])).all()
+
+The SageMaker Python SDK will automatically translate your existing workspace environment and any associated data
+processing code and datasets into a SageMaker Training job that runs on the SageMaker Training platform.
+You can also activate a persistent cache feature, which will further reduce job start up latency by caching previously
+downloaded dependency packages. This reduction in job latency is greater than the reduction in latency from using
+SageMaker managed warm pools alone. The following sections show you how to wrap your local ML code and tailor your
+experience for your use case including customizing your environment and integrating with SageMaker Experiments.
+
+See the `Run your local code as a SageMaker Training job <https://docs.aws.amazon.com/sagemaker/latest/dg/train-remote-decorator.html>`__ for detailed developer guide.
+
+Follow is the API specification for methods and classes related to remote function feature.
+
+.. toctree::
+    :maxdepth: 1
+
+    remote_function/sagemaker.remote_function.rst
+
 ***
 FAQ
 ***

doc/remote_function/sagemaker.remote_function.rst

@@ -0,0 +1,26 @@
+Remote function classes and methods specification
+=================================================
+
+
+@remote decorator
+-----------------
+
+.. automethod:: sagemaker.remote_function.client.remote
+
+
+RemoteExcutor
+-------------
+
+.. autoclass:: sagemaker.remote_function.RemoteExecutor
+    :members:
+
+
+Future
+------
+
+.. autoclass:: sagemaker.remote_function.client.Future
+    :members:
+
+.. automethod:: sagemaker.remote_function.client.list_futures
+
+.. automethod:: sagemaker.remote_function.client.get_future

src/sagemaker/clarify.py

@@ -26,7 +26,7 @@
 import tempfile
 from abc import ABC, abstractmethod
 from typing import List, Union, Dict, Optional, Any
-
+from enum import Enum
 from schema import Schema, And, Use, Or, Optional as SchemaOptional, Regex
 
 from sagemaker import image_uris, s3, utils
@@ -304,6 +304,16 @@
 )
 
 
+class DatasetType(Enum):
+    """Enum to store different dataset types supported in the Analysis config file"""
+
+    TEXTCSV = "text/csv"
+    JSONLINES = "application/jsonlines"
+    JSON = "application/json"
+    PARQUET = "application/x-parquet"
+    IMAGE = "application/x-image"
+
+
 class DataConfig:
     """Config object related to configurations of the input and output dataset."""
 
@@ -1451,7 +1461,7 @@ def _run(
                 source=self._CLARIFY_OUTPUT,
                 destination=data_config.s3_output_path,
                 output_name="analysis_result",
-                s3_upload_mode="EndOfJob",
+                s3_upload_mode=ProcessingOutputHandler.get_s3_upload_mode(analysis_config),
             )
 
             return super().run(
@@ -2171,6 +2181,33 @@ def _upload_analysis_config(analysis_config_file, s3_output_path, sagemaker_sess
     )
 
 
+class ProcessingOutputHandler:
+    """Class to handle the parameters for SagemakerProcessor.Processingoutput"""
+
+    class S3UploadMode(Enum):
+        """Enum values for different uplaod modes to s3 bucket"""
+
+        CONTINUOUS = "Continuous"
+        ENDOFJOB = "EndOfJob"
+
+    @classmethod
+    def get_s3_upload_mode(cls, analysis_config: Dict[str, Any]) -> str:
+        """Fetches s3_upload mode based on the shap_config values
+
+        Args:
+            analysis_config (dict): dict Config following the analysis_config.json format
+
+        Returns:
+            The s3_upload_mode type for the processing output.
+        """
+        dataset_type = analysis_config["dataset_type"]
+        return (
+            ProcessingOutputHandler.S3UploadMode.CONTINUOUS.value
+            if dataset_type == DatasetType.IMAGE.value
+            else ProcessingOutputHandler.S3UploadMode.ENDOFJOB.value
+        )
+
+
 def _set(value, key, dictionary):
     """Sets dictionary[key] = value if value is not None."""
     if value is not None:

src/sagemaker/image_uri_config/tensorflow.json

@@ -2021,7 +2021,8 @@
             "2.8": "2.8.0",
             "2.9": "2.9.2",
             "2.10": "2.10.1",
-            "2.11": "2.11.0"
+            "2.11": "2.11.0",
+            "2.12": "2.12.0"
         },
         "versions": {
             "1.10.0": {
@@ -3755,6 +3756,37 @@
                     "us-west-2": "763104351884"
                 },
                 "repository": "tensorflow-training"
+            },
+            "2.12.0": {
+                "py_versions": [
+                    "py310"
+                ],
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ap-southeast-3": "907027046896",
+                    "ap-southeast-4": "457447274322",
+                    "ca-central-1": "763104351884",
+                    "eu-central-1": "763104351884",
+                    "eu-north-1": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "tensorflow-training"
             }
         }
     }

src/sagemaker/image_uris.py

@@ -369,16 +369,15 @@ def _config_for_framework_and_scope(framework, image_scope, accelerator_type=Non
 
 def _validate_instance_deprecation(framework, instance_type, version):
     """Check if instance type is deprecated for a certain framework with a certain version"""
-    if (
-        framework == "pytorch"
-        and _get_instance_type_family(instance_type) == "p2"
-        and Version(version) >= Version("1.13")
-    ):
-        raise ValueError(
-            "P2 instances have been deprecated for sagemaker jobs with PyTorch 1.13 and above. "
-            "For information about supported instance types please refer to "
-            "https://aws.amazon.com/sagemaker/pricing/"
-        )
+    if _get_instance_type_family(instance_type) == "p2":
+        if (framework == "pytorch" and Version(version) >= Version("1.13")) or (
+            framework == "tensorflow" and Version(version) >= Version("2.12")
+        ):
+            raise ValueError(
+                "P2 instances have been deprecated for sagemaker jobs starting PyTorch 1.13 and TensorFlow 2.12"
+                "For information about supported instance types please refer to "
+                "https://aws.amazon.com/sagemaker/pricing/"
+            )
 
 
 def _validate_for_suppported_frameworks_and_instance_type(framework, instance_type):