Merge branch 'master' into master

Author: jerrypeng7773

doc/amazon_sagemaker_model_building_pipeline.rst

@@ -239,7 +239,7 @@ Referable Property List:
                 ProcessingOutput(output_name="train", source="/opt/ml/processing/train"),
             ],
             code="./local/preprocess.py",
-            job_arguments=["--input-data", "s3://my-input"]
+            arguments=["--input-data", "s3://my-input"]
         ),
     )
 
@@ -496,7 +496,7 @@ A :class:`sagemaker.workflow.properties.PropertyFile` is designed to store infor
             ProcessingOutput(output_name="hyperparam", source="/opt/ml/processing/evaluation"),
         ],
         code="./local/preprocess.py",
-        job_arguments=["--input-data", "s3://my-input"],
+        arguments=["--input-data", "s3://my-input"],
     )
 
     hyperparam_report = PropertyFile(
@@ -577,7 +577,7 @@ There are eight types of condition are supported, they are:
             ProcessingOutput(output_name="hyperparam", source="/opt/ml/processing/evaluation"),
         ],
         code="./local/preprocess.py",
-        job_arguments=["--input-data", "s3://my-input"],
+        arguments=["--input-data", "s3://my-input"],
     )
 
     eval_report = PropertyFile(
@@ -668,7 +668,7 @@ Use :class:`sagemaker.workflow.functions.JsonGet` to extract a Json property fro
             ProcessingOutput(output_name="hyperparam", source="/opt/ml/processing/evaluation"),
         ],
         code="./local/preprocess.py",
-        job_arguments=["--input-data", "s3://my-input"],
+        arguments=["--input-data", "s3://my-input"],
     )
 
     hyperparam_report = PropertyFile(

src/sagemaker/image_uri_config/mxnet.json

@@ -10,7 +10,8 @@
             "1.4": "1.4.1",
             "1.6": "1.6.0",
             "1.7": "1.7.0",
-            "1.8": "1.8.0"
+            "1.8": "1.8.0",
+            "1.9": "1.9.0"
         },
         "versions": {
             "0.12.1": {
@@ -346,6 +347,37 @@
                 },
                 "repository": "mxnet-training",
                 "py_versions": ["py37"]
+            },
+            "1.9.0": {
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-north-1": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "mxnet-training",
+                "py_versions": ["py38"]
             }
         }
     },
@@ -358,7 +390,10 @@
             "1.2": "1.2.1",
             "1.3": "1.3.0",
             "1.4": "1.4.1",
-            "1.6": "1.6.0"
+            "1.6": "1.6.0",
+            "1.7": "1.7.0",
+            "1.8": "1.8.0",
+            "1.9": "1.9.0"
         },
         "versions": {
             "0.12.1": {
@@ -694,6 +729,37 @@
                 },
                 "repository": "mxnet-inference",
                 "py_versions": ["py37"]
+            },
+            "1.9.0": {
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-north-1": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "mxnet-inference",
+                "py_versions": ["py38"]
             }
         }
     },

src/sagemaker/model.py

@@ -501,13 +501,14 @@ def _upload_code(self, key_prefix: str, repack: bool = False) -> None:
             if is_pipeline_variable(self.model_data):
                 # model is not yet there, defer repacking to later during pipeline execution
                 if not isinstance(self.sagemaker_session, PipelineSession):
-                    # TODO: link the doc in the warning once ready
                     logging.warning(
                         "The model_data is a Pipeline variable of type %s, "
                         "which should be used under `PipelineSession` and "
                         "leverage `ModelStep` to create or register model. "
                         "Otherwise some functionalities e.g. "
-                        "runtime repack may be missing",
+                        "runtime repack may be missing. For more, see: "
+                        "https://sagemaker.readthedocs.io/en/stable/"
+                        "amazon_sagemaker_model_building_pipeline.html#model-step",
                         type(self.model_data),
                     )
                     return

src/sagemaker/pipeline.py

@@ -278,6 +278,7 @@ def register(
         description: Optional[str] = None,
         drift_check_baselines: Optional[DriftCheckBaselines] = None,
         customer_metadata_properties: Optional[Dict[str, str]] = None,
+        domain: Optional[str] = None,
     ):
         """Creates a model package for creating SageMaker models or listing on Marketplace.
 
@@ -305,6 +306,8 @@ def register(
             drift_check_baselines (DriftCheckBaselines): DriftCheckBaselines object (default: None).
             customer_metadata_properties (dict[str, str]): A dictionary of key-value paired
                 metadata properties (default: None).
+            domain (str): Domain values can be "COMPUTER_VISION", "NATURAL_LANGUAGE_PROCESSING",
+                "MACHINE_LEARNING" (default: None).
 
         Returns:
             A `sagemaker.model.ModelPackage` instance.
@@ -335,6 +338,7 @@ def register(
             container_def_list=container_def,
             drift_check_baselines=drift_check_baselines,
             customer_metadata_properties=customer_metadata_properties,
+            domain=domain,
         )
 
         self.sagemaker_session.create_model_package_from_containers(**model_pkg_args)

src/sagemaker/tensorflow/estimator.py

@@ -25,6 +25,7 @@
 from sagemaker.tensorflow.model import TensorFlowModel
 from sagemaker.transformer import Transformer
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
+from sagemaker.workflow import is_pipeline_variable
 
 logger = logging.getLogger("sagemaker")
 
@@ -378,6 +379,9 @@ def _default_s3_path(self, directory, mpi=False):
         if mpi:
             return "/opt/ml/model"
         if self._current_job_name:
+            if is_pipeline_variable(self.output_path):
+                output_path = "s3://{}".format(self.sagemaker_session.default_bucket())
+                return s3.s3_path_join(output_path, self._current_job_name, directory)
             return s3.s3_path_join(self.output_path, self._current_job_name, directory)
         return None
 

src/sagemaker/tensorflow/model.py

@@ -362,13 +362,14 @@ def prepare_container_def(
             if isinstance(self.sagemaker_session, PipelineSession):
                 self.sagemaker_session.context.need_runtime_repack.add(id(self))
             else:
-                # TODO: link the doc in the warning once ready
                 logging.warning(
                     "The model_data is a Pipeline variable of type %s, "
                     "which should be used under `PipelineSession` and "
                     "leverage `ModelStep` to create or register model. "
                     "Otherwise some functionalities e.g. "
-                    "runtime repack may be missing",
+                    "runtime repack may be missing. For more, see: "
+                    "https://sagemaker.readthedocs.io/en/stable/"
+                    "amazon_sagemaker_model_building_pipeline.html#model-step",
                     type(self.model_data),
                 )
             model_data = self.model_data

src/sagemaker/transformer.py

@@ -191,7 +191,7 @@ def transform(
                 Only meaningful when wait is ``True`` (default: ``True``).
         """
         local_mode = self.sagemaker_session.local_mode
-        if not local_mode and not data.startswith("s3://"):
+        if not local_mode and not is_pipeline_variable(data) and not data.startswith("s3://"):
             raise ValueError("Invalid S3 URI: {}".format(data))
 
         if job_name is not None:

src/sagemaker/workflow/model_step.py

@@ -111,7 +111,6 @@ def __init__(
                 The display name provides better UI readability. (default: None).
             description (str): The description of the `ModelStep` (default: None).
         """
-        # TODO: add a doc link in error message once ready
         from sagemaker.workflow.utilities import validate_step_args_input
 
         validate_step_args_input(
@@ -121,7 +120,8 @@ def __init__(
                 Session.create_model_package_from_containers.__name__,
             },
             error_message="The step_args of ModelStep must be obtained from model.create() "
-            "or model.register().",
+            "or model.register(). For more, see: https://sagemaker.readthedocs.io/en/stable/"
+            "amazon_sagemaker_model_building_pipeline.html#model-step",
         )
         if not (step_args.create_model_request is None) ^ (
             step_args.create_model_package_request is None

src/sagemaker/workflow/step_collections.py

@@ -253,12 +253,13 @@ def __init__(
         steps.append(register_model_step)
         self.steps = steps
 
-        # TODO: add public document link here once ready
         warnings.warn(
             (
                 "We are deprecating the use of RegisterModel. "
                 "Instead, please use the ModelStep, which simply takes in the step arguments "
-                "generated by model.register()."
+                "generated by model.register(). For more, see: "
+                "https://sagemaker.readthedocs.io/en/stable/"
+                "amazon_sagemaker_model_building_pipeline.html#model-step"
             ),
             DeprecationWarning,
         )

src/sagemaker/workflow/steps.py

@@ -444,12 +444,13 @@ def __init__(
 
         self._properties = Properties(path=f"Steps.{name}", shape_name="DescribeModelOutput")
 
-        # TODO: add public document link here once ready
         warnings.warn(
             (
                 "We are deprecating the use of CreateModelStep. "
                 "Instead, please use the ModelStep, which simply takes in the step arguments "
-                "generated by model.create()."
+                "generated by model.create(). For more, see: "
+                "https://sagemaker.readthedocs.io/en/stable/"
+                "amazon_sagemaker_model_building_pipeline.html#model-step"
             ),
             DeprecationWarning,
         )

tests/conftest.py

@@ -181,6 +181,8 @@ def mxnet_inference_py_version(mxnet_inference_version, request):
         return request.param
     elif Version(mxnet_inference_version) == Version("1.8.0"):
         return "py37"
+    elif Version(mxnet_inference_version) == Version("1.9.0"):
+        return "py38"
     else:
         return "py3"
 
@@ -191,6 +193,8 @@ def mxnet_training_py_version(mxnet_training_version, request):
         return request.param
     elif Version(mxnet_training_version) == Version("1.8.0"):
         return "py37"
+    elif Version(mxnet_training_version) == Version("1.9.0"):
+        return "py38"
     else:
         return "py3"
 

tests/integ/sagemaker/workflow/test_model_steps.py

@@ -608,6 +608,9 @@ def test_model_registration_with_tensorflow_model_with_pipeline_model(
     )
     inputs = TrainingInput(s3_data=input_path)
     instance_count = ParameterInteger(name="InstanceCount", default_value=1)
+    output_path = ParameterString(
+        name="OutputPath", default_value=f"s3://{pipeline_session.default_bucket()}"
+    )
 
     # If image_uri is not provided, the instance_type should not be a pipeline variable
     # since instance_type is used to retrieve image_uri in compile time (PySDK)
@@ -619,6 +622,7 @@ def test_model_registration_with_tensorflow_model_with_pipeline_model(
         framework_version=tf_full_version,
         py_version=tf_full_py_version,
         sagemaker_session=pipeline_session,
+        output_path=output_path,
     )
     train_step_args = tensorflow_estimator.fit(inputs=inputs)
     step_train = TrainingStep(
@@ -648,7 +652,7 @@ def test_model_registration_with_tensorflow_model_with_pipeline_model(
     )
     pipeline = Pipeline(
         name=pipeline_name,
-        parameters=[instance_count],
+        parameters=[instance_count, output_path],
         steps=[step_train, step_register_model],
         sagemaker_session=pipeline_session,
     )

tests/unit/sagemaker/workflow/test_training_step.py

@@ -15,6 +15,7 @@
 import os
 import json
 from mock import Mock, PropertyMock
+import re
 
 import pytest
 import warnings

tests/unit/sagemaker/workflow/test_transform_step.py

@@ -89,10 +89,10 @@ def test_transform_step_with_transformer(pipeline_session):
         output_path=f"s3://{pipeline_session.default_bucket()}/Transform",
         sagemaker_session=pipeline_session,
     )
-
-    transform_inputs = TransformInput(
-        data=f"s3://{pipeline_session.default_bucket()}/batch-data",
+    data = ParameterString(
+        name="Data", default_value=f"s3://{pipeline_session.default_bucket()}/batch-data"
     )
+    transform_inputs = TransformInput(data=data)
 
     with warnings.catch_warnings(record=True) as w:
         step_args = transformer.transform(
@@ -120,10 +120,11 @@ def test_transform_step_with_transformer(pipeline_session):
     pipeline = Pipeline(
         name="MyPipeline",
         steps=[step],
-        parameters=[model_name],
+        parameters=[model_name, data],
         sagemaker_session=pipeline_session,
     )
     step_args.args["ModelName"] = model_name.expr
+    step_args.args["TransformInput"]["DataSource"]["S3DataSource"]["S3Uri"] = data.expr
     assert json.loads(pipeline.definition())["Steps"][0] == {
         "Name": "MyTransformStep",
         "Type": "Transform",

tests/unit/test_mxnet.py

@@ -747,7 +747,7 @@ def test_model_prepare_container_def_no_instance_type_or_image(
 
 
 def test_attach(sagemaker_session, mxnet_training_version, mxnet_training_py_version):
-    if mxnet_training_py_version == "py37":
+    if mxnet_training_py_version in ["py37", "py38"]:
         training_image = "1.dkr.ecr.us-west-2.amazonaws.com/mxnet-training:{1}-cpu-{0}".format(
             mxnet_training_py_version, mxnet_training_version
         )