Merge pull request #2 from aws/master

merge master

Author: cj-zhang

CHANGELOG.md

@@ -1,5 +1,63 @@
 # Changelog
 
+## v2.192.0 (2023-10-11)
+
+### Features
+
+ * jumpstart estimator enable infra check flag
+ * jumpstart default payloads
+ * allow non-python files in job dependencies
+ * allow configuring docker container in local mode
+
+### Bug Fixes and Other Changes
+
+ * js tagging s3 prefix
+ * Batch transform: Add support for split_type == "None" in local mode
+ * use correct line endings and s3 uris on windows
+ * Fixed bug in _create_training_details
+ * DJL Neuronx 0.24.0
+
+### Documentation Changes
+
+ * Include FeatureGroup's load_feature_definitions API documentation
+
+## v2.191.0 (2023-10-05)
+
+### Features
+
+ * Selective Step Execution milestone 2 features
+ * feature-processor extra data sources support
+
+## v2.190.0 (2023-10-04)
+
+### Features
+
+ * Add support for in-memory feature groups and collection type features in Feature Store.
+
+### Bug Fixes and Other Changes
+
+ * chore: xfail resource in use failure for specific test
+ * Add missing API docs for processors
+
+### Documentation Changes
+
+ * Bring back (de)serializers documentation
+ * Add missing AirFlow operators + link to airflow documentation
+
+## v2.189.0 (2023-10-03)
+
+### Features
+
+ * add feature processor APIs to public doc
+ * s3 prefix model data for JumpStartModel
+ * Model Package support for updating approval
+
+### Bug Fixes and Other Changes
+
+ * Add bucket owner check
+ * transform step  unit test
+ * Release TGI 1.1.0 Image
+
 ## v2.188.0 (2023-09-26)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.188.1.dev0
+2.192.1.dev0

doc/api/inference/deserializers.rst

@@ -1,8 +1,12 @@
 Deserializers
 -------------
 
-.. automodule:: sagemaker.deserializers
+.. automodule:: sagemaker.base_deserializers
     :members:
     :undoc-members:
     :show-inheritance:
 
+.. automodule:: sagemaker.deserializers
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/api/inference/serializers.rst

@@ -1,8 +1,12 @@
 Serializers
 -----------
 
-.. automodule:: sagemaker.serializers
+.. automodule:: sagemaker.base_serializers
     :members:
     :undoc-members:
     :show-inheritance:
 
+.. automodule:: sagemaker.serializers
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/api/prep_data/feature_store.rst

@@ -6,7 +6,6 @@ Feature Group
 
 .. autoclass:: sagemaker.feature_store.feature_group.FeatureGroup
     :members:
-    :exclude-members: load_feature_definitions
     :show-inheritance:
 
 .. autoclass:: sagemaker.feature_store.feature_group.AthenaQuery
@@ -41,6 +40,25 @@ Feature Definition
     :members:
     :show-inheritance:
 
+.. autoclass:: sagemaker.feature_store.feature_definition.CollectionTypeEnum
+    :members:
+    :show-inheritance:
+
+.. autoclass:: sagemaker.feature_store.feature_definition.CollectionType
+    :members:
+    :show-inheritance:
+
+.. autoclass:: sagemaker.feature_store.feature_definition.ListCollectionType
+    :members:
+    :show-inheritance:
+
+.. autoclass:: sagemaker.feature_store.feature_definition.SetCollectionType
+    :members:
+    :show-inheritance:
+
+.. autoclass:: sagemaker.feature_store.feature_definition.VectorCollectionType
+    :members:
+    :show-inheritance:
 
 Inputs
 ******
@@ -77,6 +95,10 @@ Inputs
     :members:
     :show-inheritance:
 
+.. autoclass:: sagemaker.feature_store.inputs.OnlineStoreStorageTypeEnum
+    :members:
+    :show-inheritance:
+
 .. autoclass:: sagemaker.feature_store.inputs.ResourceEnum
     :members:
     :show-inheritance:
@@ -120,3 +142,41 @@ Feature Store
 .. autoclass:: sagemaker.feature_store.feature_store.FeatureStore
     :members:
     :show-inheritance:
+
+
+@feature_processor Decorator
+****************************
+
+.. autodecorator:: sagemaker.feature_store.feature_processor.feature_processor
+
+
+Feature Processor Data Source
+*****************************
+
+.. autoclass:: sagemaker.feature_store.feature_processor.FeatureGroupDataSource
+    :members:
+    :show-inheritance:
+
+.. autoclass:: sagemaker.feature_store.feature_processor.CSVDataSource
+    :members:
+    :show-inheritance:
+
+.. autoclass:: sagemaker.feature_store.feature_processor.ParquetDataSource
+    :members:
+    :show-inheritance:
+
+
+Feature Processor Scheduler
+***************************
+
+.. automethod:: sagemaker.feature_store.feature_processor.to_pipeline
+
+.. automethod:: sagemaker.feature_store.feature_processor.schedule
+
+.. automethod:: sagemaker.feature_store.feature_processor.execute
+
+.. automethod:: sagemaker.feature_store.feature_processor.delete_schedule
+
+.. automethod:: sagemaker.feature_store.feature_processor.describe
+
+.. automethod:: sagemaker.feature_store.feature_processor.list_pipelines

doc/conf.py

@@ -109,6 +109,8 @@
 # autosectionlabel
 autosectionlabel_prefix_document = True
 
+autodoc_mock_imports = ["pyspark", "feature_store_pyspark", "py4j"]
+
 
 def setup(app):
     sys.stdout.write("Generating JumpStart model table...")

doc/frameworks/huggingface/sagemaker.huggingface.rst

@@ -32,3 +32,11 @@ Hugging Face Predictor
     :members:
     :undoc-members:
     :show-inheritance:
+
+Hugging Face Processor
+----------------------
+
+.. autoclass:: sagemaker.huggingface.processing.HuggingFaceProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/frameworks/mxnet/sagemaker.mxnet.rst

@@ -3,25 +3,33 @@ MXNet Classes
 
 
 MXNet Estimator
----------------------------
+---------------
 
 .. autoclass:: sagemaker.mxnet.estimator.MXNet
     :members:
     :undoc-members:
     :show-inheritance:
 
 MXNet Model
----------------------------
+-----------
 
 .. autoclass:: sagemaker.mxnet.model.MXNetModel
     :members:
     :undoc-members:
     :show-inheritance:
 
 MXNet Predictor
----------------------------
+---------------
 
 .. autoclass:: sagemaker.mxnet.model.MXNetPredictor
     :members:
     :undoc-members:
     :show-inheritance:
+
+MXNet Processor
+---------------
+
+.. autoclass:: sagemaker.mxnet.processing.MXNetProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/frameworks/pytorch/sagemaker.pytorch.rst

@@ -24,3 +24,11 @@ PyTorch Predictor
     :members:
     :undoc-members:
     :show-inheritance:
+
+PyTorch Processor
+-----------------
+
+.. autoclass:: sagemaker.pytorch.processing.PyTorchProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/frameworks/tensorflow/sagemaker.tensorflow.rst

@@ -33,3 +33,12 @@ TensorFlow Serving Predictor
     :members:
     :undoc-members:
     :show-inheritance:
+
+
+TensorFlow Processor
+--------------------
+
+.. autoclass:: sagemaker.tensorflow.processing.TensorFlowProcessor
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/overview.rst

@@ -1560,9 +1560,20 @@ You can install necessary dependencies for this feature using pip.
 Additionally, Local Mode also requires Docker Compose V2. Follow the guidelines in https://docs.docker.com/compose/install/ to install.
 Make sure to have a Compose Version compatible with your Docker Engine installation. Check Docker Engine release notes https://docs.docker.com/engine/release-notes to find a compatible version.
 
-If you want to keep everything local, and not use Amazon S3 either, you can enable "local code" in one of two ways:
+Local mode configuration
+========================
 
-- Create a file at ``~/.sagemaker/config.yaml`` that contains:
+The local mode uses a YAML configuration file located at ``~/.sagemaker/config.yaml`` to define the default values that are automatically passed to the ``config`` attribute of ``LocalSession``. This is an example of the configuration, for the full schema, see `sagemaker.config.config_schema.SAGEMAKER_PYTHON_SDK_LOCAL_MODE_CONFIG_SCHEMA <https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/config/config_schema.py>`_.
+
+.. code:: yaml
+
+    local:
+        local_code: true # Using everything locally
+        region_name: "us-west-2" # Name of the region
+        container_config: # Additional docker container config
+            shm_size: "128M
+
+If you want to keep everything local, and not use Amazon S3 either, you can enable "local code" in one of two ways:
 
 .. code:: yaml
 
@@ -1583,6 +1594,9 @@ If you want to keep everything local, and not use Amazon S3 either, you can enab
 .. note::
     If you enable "local code," then you cannot use the ``dependencies`` parameter in your estimator or model.
 
+Activating local mode by ``instance_type`` argument
+====================================================
+
 We can take the example in  `Using Estimators <#using-estimators>`__ , and use either ``local`` or ``local_gpu`` as the instance type.
 
 .. code:: python

doc/workflows/airflow/using_workflow.rst

@@ -22,6 +22,16 @@ the configuration dictionary in the SageMaker Python SDK. Currently, the followi
 * ``SageMakerTransformOperator``
 * ``SageMakerEndpointConfigOperator``
 * ``SageMakerEndpointOperator``
+* ``SageMakerProcessingOperator``
+* ``SageMakerDeleteModelOperator``
+* ``SageMakerStartPipelineOperator``
+* ``SageMakerStopPipelineOperator``
+* ``SageMakerRegisterModelVersionOperator``
+* ``SageMakerAutoMLOperator``
+* ``SageMakerCreateExperimentOperator``
+
+You can look into `the Apache Airflow documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/operators/sagemaker.html>`_ for the full
+list of operators with examples.
 
 2. PythonOperator: Airflow built-in operator that executes Python callables. You can use the PythonOperator to execute
 operations in the SageMaker Python SDK to create a SageMaker workflow.

src/sagemaker/base_predictor.py

@@ -14,7 +14,7 @@
 from __future__ import print_function, absolute_import
 
 import abc
-from typing import Any, Tuple
+from typing import Any, Optional, Tuple, Union
 
 from sagemaker.deprecations import (
     deprecated_class,
@@ -32,6 +32,9 @@
     StreamDeserializer,
     StringDeserializer,
 )
+from sagemaker.jumpstart.payload_utils import PayloadSerializer
+from sagemaker.jumpstart.types import JumpStartSerializablePayload
+from sagemaker.jumpstart.utils import get_jumpstart_content_bucket
 from sagemaker.model_monitor import (
     DataCaptureConfig,
     DefaultModelMonitor,
@@ -201,20 +204,44 @@ def _create_request_args(
         custom_attributes=None,
     ):
         """Placeholder docstring"""
+
+        jumpstart_serialized_data: Optional[Union[str, bytes]] = None
+        jumpstart_accept: Optional[str] = None
+        jumpstart_content_type: Optional[str] = None
+
+        if isinstance(data, JumpStartSerializablePayload):
+            s3_client = self.sagemaker_session.s3_client
+            region = self.sagemaker_session._region_name
+            bucket = get_jumpstart_content_bucket(region)
+
+            jumpstart_serialized_data = PayloadSerializer(
+                bucket=bucket, region=region, s3_client=s3_client
+            ).serialize(data)
+            jumpstart_content_type = data.content_type
+            jumpstart_accept = data.accept
+
         args = dict(initial_args) if initial_args else {}
 
         if "EndpointName" not in args:
             args["EndpointName"] = self.endpoint_name
 
         if "ContentType" not in args:
-            args["ContentType"] = (
-                self.content_type
-                if isinstance(self.content_type, str)
-                else ", ".join(self.content_type)
-            )
+            if isinstance(data, JumpStartSerializablePayload) and jumpstart_content_type:
+                args["ContentType"] = jumpstart_content_type
+            else:
+                args["ContentType"] = (
+                    self.content_type
+                    if isinstance(self.content_type, str)
+                    else ", ".join(self.content_type)
+                )
 
         if "Accept" not in args:
-            args["Accept"] = self.accept if isinstance(self.accept, str) else ", ".join(self.accept)
+            if isinstance(data, JumpStartSerializablePayload) and jumpstart_accept:
+                args["Accept"] = jumpstart_accept
+            else:
+                args["Accept"] = (
+                    self.accept if isinstance(self.accept, str) else ", ".join(self.accept)
+                )
 
         if target_model:
             args["TargetModel"] = target_model
@@ -228,7 +255,11 @@ def _create_request_args(
         if custom_attributes:
             args["CustomAttributes"] = custom_attributes
 
-        data = self.serializer.serialize(data)
+        data = (
+            jumpstart_serialized_data
+            if isinstance(data, JumpStartSerializablePayload) and jumpstart_serialized_data
+            else self.serializer.serialize(data)
+        )
 
         args["Body"] = data
         return args