Merge branch 'master' into fix/make-instance-type-optional

Author: rahven14

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## v2.92.2 (2022-05-31)
+
+### Bug Fixes and Other Changes
+
+ * turn off Pipeline Parameter inheritance from python primitives
+ * Add more validations for pipeline step new interfaces
+ * Changed method description per AWS request
+
+## v2.92.1 (2022-05-26)
+
+### Bug Fixes and Other Changes
+
+ * pin protobuf to < 4.0 to fix breaking change
+
 ## v2.92.0 (2022-05-26)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.92.1.dev0
+2.92.3.dev0

src/sagemaker/clarify.py

@@ -827,7 +827,7 @@ def run_pre_training_bias(
                 "`TVD <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-total-variation-distance.html>`_",
                 "`KS <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-kolmogorov-smirnov.html>`_",
                 "`CDDL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-cddl.html>`_"].
-                Defaults to computing all.
+                Defaults to str "all" to run all metrics if left unspecified.
             wait (bool): Whether the call should wait until the job completes (default: True).
             logs (bool): Whether to show the logs produced by the job.
                 Only meaningful when ``wait`` is True (default: True).
@@ -906,7 +906,7 @@ def run_post_training_bias(
                 "`CDDPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-cddpl.html>`_
                 ", "`TE <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-te.html>`_",
                 "`FT <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-ft.html>`_"].
-                Defaults to computing all.
+                Defaults to str "all" to run all metrics if left unspecified.
             wait (bool): Whether the call should wait until the job completes (default: True).
             logs (bool): Whether to show the logs produced by the job.
                 Only meaningful when ``wait`` is True (default: True).
@@ -989,7 +989,7 @@ def run_bias(
                 "`TVD <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-total-variation-distance.html>`_",
                 "`KS <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-kolmogorov-smirnov.html>`_",
                 "`CDDL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-cddl.html>`_"].
-                Defaults to computing all.
+                Defaults to str "all" to run all metrics if left unspecified.
             post_training_methods (str or list[str]): Selector of a subset of potential metrics:
                 ["`DPPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dppl.html>`_"
                 , "`DI <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-di.html>`_",
@@ -1002,7 +1002,7 @@ def run_bias(
                 "`CDDPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-cddpl.html>`_
                 ", "`TE <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-te.html>`_",
                 "`FT <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-ft.html>`_"].
-                Defaults to computing all.
+                Defaults to str "all" to run all metrics if left unspecified.
             wait (bool): Whether the call should wait until the job completes (default: True).
             logs (bool): Whether to show the logs produced by the job.
                 Only meaningful when ``wait`` is True (default: True).

src/sagemaker/estimator.py

@@ -483,6 +483,7 @@ def __init__(
         if (
             not self.sagemaker_session.local_mode
             and output_path
+            and not is_pipeline_variable(output_path)
             and output_path.startswith("file://")
         ):
             raise RuntimeError("file:// output paths are only supported in Local Mode")
@@ -1340,6 +1341,11 @@ def register(
         Returns:
             str: A string of SageMaker Model Package ARN.
         """
+        if isinstance(self.sagemaker_session, PipelineSession):
+            raise TypeError(
+                "estimator.register does not support PipelineSession at this moment. "
+                "Please use model.register with PipelineSession if you're using the ModelStep."
+            )
         default_name = name_from_base(self.base_job_name)
         model_name = model_name or default_name
         if compile_model_family is not None:

src/sagemaker/fw_utils.py

@@ -428,6 +428,10 @@ def warn_if_parameter_server_with_multi_gpu(training_instance_type, distribution
     """
     if training_instance_type == "local" or distribution is None:
         return
+    if is_pipeline_variable(training_instance_type):
+        # The training_instance_type is not available in compile time.
+        # Rather, it's given in Pipeline execution time
+        return
 
     is_multi_gpu_instance = (
         training_instance_type == "local_gpu"
@@ -485,6 +489,10 @@ def validate_smdistributed(
     if "smdistributed" not in distribution:
         # Distribution strategy other than smdistributed is selected
         return
+    if is_pipeline_variable(instance_type):
+        # The instance_type is not available in compile time.
+        # Rather, it's given in Pipeline execution time
+        return
 
     # distribution contains smdistributed
     smdistributed = distribution["smdistributed"]

src/sagemaker/image_uri_config/mxnet.json

@@ -10,7 +10,8 @@
             "1.4": "1.4.1",
             "1.6": "1.6.0",
             "1.7": "1.7.0",
-            "1.8": "1.8.0"
+            "1.8": "1.8.0",
+            "1.9": "1.9.0"
         },
         "versions": {
             "0.12.1": {
@@ -346,6 +347,37 @@
                 },
                 "repository": "mxnet-training",
                 "py_versions": ["py37"]
+            },
+            "1.9.0": {
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-north-1": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "mxnet-training",
+                "py_versions": ["py38"]
             }
         }
     },
@@ -358,7 +390,10 @@
             "1.2": "1.2.1",
             "1.3": "1.3.0",
             "1.4": "1.4.1",
-            "1.6": "1.6.0"
+            "1.6": "1.6.0",
+            "1.7": "1.7.0",
+            "1.8": "1.8.0",
+            "1.9": "1.9.0"
         },
         "versions": {
             "0.12.1": {
@@ -694,6 +729,37 @@
                 },
                 "repository": "mxnet-inference",
                 "py_versions": ["py37"]
+            },
+            "1.9.0": {
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-north-1": "763104351884",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "mxnet-inference",
+                "py_versions": ["py38"]
             }
         }
     },

src/sagemaker/model.py

@@ -500,13 +500,14 @@ def _upload_code(self, key_prefix: str, repack: bool = False) -> None:
             if is_pipeline_variable(self.model_data):
                 # model is not yet there, defer repacking to later during pipeline execution
                 if not isinstance(self.sagemaker_session, PipelineSession):
-                    # TODO: link the doc in the warning once ready
                     logging.warning(
                         "The model_data is a Pipeline variable of type %s, "
                         "which should be used under `PipelineSession` and "
                         "leverage `ModelStep` to create or register model. "
                         "Otherwise some functionalities e.g. "
-                        "runtime repack may be missing",
+                        "runtime repack may be missing. For more, see: "
+                        "https://sagemaker.readthedocs.io/en/stable/"
+                        "amazon_sagemaker_model_building_pipeline.html#model-step",
                         type(self.model_data),
                     )
                     return

src/sagemaker/pipeline.py

@@ -278,6 +278,7 @@ def register(
         description: Optional[str] = None,
         drift_check_baselines: Optional[DriftCheckBaselines] = None,
         customer_metadata_properties: Optional[Dict[str, str]] = None,
+        domain: Optional[str] = None,
     ):
         """Creates a model package for creating SageMaker models or listing on Marketplace.
 
@@ -305,6 +306,8 @@ def register(
             drift_check_baselines (DriftCheckBaselines): DriftCheckBaselines object (default: None).
             customer_metadata_properties (dict[str, str]): A dictionary of key-value paired
                 metadata properties (default: None).
+            domain (str): Domain values can be "COMPUTER_VISION", "NATURAL_LANGUAGE_PROCESSING",
+                "MACHINE_LEARNING" (default: None).
 
         Returns:
             A `sagemaker.model.ModelPackage` instance.
@@ -337,6 +340,7 @@ def register(
             container_def_list=container_def,
             drift_check_baselines=drift_check_baselines,
             customer_metadata_properties=customer_metadata_properties,
+            domain=domain,
         )
 
         self.sagemaker_session.create_model_package_from_containers(**model_pkg_args)

src/sagemaker/session.py

@@ -591,7 +591,7 @@ def submit(request):
             LOGGER.debug("train request: %s", json.dumps(request, indent=4))
             self.sagemaker_client.create_training_job(**request)
 
-        self._intercept_create_request(train_request, submit)
+        self._intercept_create_request(train_request, submit, self.train.__name__)
 
     def _get_train_request(  # noqa: C901
         self,
@@ -922,7 +922,7 @@ def submit(request):
             LOGGER.debug("process request: %s", json.dumps(request, indent=4))
             self.sagemaker_client.create_processing_job(**request)
 
-        self._intercept_create_request(process_request, submit)
+        self._intercept_create_request(process_request, submit, self.process.__name__)
 
     def _get_process_request(
         self,
@@ -2099,7 +2099,7 @@ def submit(request):
             LOGGER.debug("tune request: %s", json.dumps(request, indent=4))
             self.sagemaker_client.create_hyper_parameter_tuning_job(**request)
 
-        self._intercept_create_request(tune_request, submit)
+        self._intercept_create_request(tune_request, submit, self.create_tuning_job.__name__)
 
     def _get_tuning_request(
         self,
@@ -2569,7 +2569,7 @@ def submit(request):
             LOGGER.debug("Transform request: %s", json.dumps(request, indent=4))
             self.sagemaker_client.create_transform_job(**request)
 
-        self._intercept_create_request(transform_request, submit)
+        self._intercept_create_request(transform_request, submit, self.transform.__name__)
 
     def _create_model_request(
         self,

src/sagemaker/tensorflow/estimator.py

@@ -25,6 +25,7 @@
 from sagemaker.tensorflow.model import TensorFlowModel
 from sagemaker.transformer import Transformer
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
+from sagemaker.workflow import is_pipeline_variable
 
 logger = logging.getLogger("sagemaker")
 
@@ -378,6 +379,9 @@ def _default_s3_path(self, directory, mpi=False):
         if mpi:
             return "/opt/ml/model"
         if self._current_job_name:
+            if is_pipeline_variable(self.output_path):
+                output_path = "s3://{}".format(self.sagemaker_session.default_bucket())
+                return s3.s3_path_join(output_path, self._current_job_name, directory)
             return s3.s3_path_join(self.output_path, self._current_job_name, directory)
         return None
 

src/sagemaker/tensorflow/model.py

@@ -362,13 +362,14 @@ def prepare_container_def(
             if isinstance(self.sagemaker_session, PipelineSession):
                 self.sagemaker_session.context.need_runtime_repack.add(id(self))
             else:
-                # TODO: link the doc in the warning once ready
                 logging.warning(
                     "The model_data is a Pipeline variable of type %s, "
                     "which should be used under `PipelineSession` and "
                     "leverage `ModelStep` to create or register model. "
                     "Otherwise some functionalities e.g. "
-                    "runtime repack may be missing",
+                    "runtime repack may be missing. For more, see: "
+                    "https://sagemaker.readthedocs.io/en/stable/"
+                    "amazon_sagemaker_model_building_pipeline.html#model-step",
                     type(self.model_data),
                 )
             model_data = self.model_data

src/sagemaker/transformer.py

@@ -191,7 +191,7 @@ def transform(
                 Only meaningful when wait is ``True`` (default: ``True``).
         """
         local_mode = self.sagemaker_session.local_mode
-        if not local_mode and not data.startswith("s3://"):
+        if not local_mode and not is_pipeline_variable(data) and not data.startswith("s3://"):
             raise ValueError("Invalid S3 URI: {}".format(data))
 
         if job_name is not None:

src/sagemaker/workflow/__init__.py

@@ -12,14 +12,8 @@
 # language governing permissions and limitations under the License.
 """Defines Types etc. used in workflow."""
 from __future__ import absolute_import
-from typing import Union
 
 from sagemaker.workflow.entities import Expression
-from sagemaker.workflow.execution_variables import ExecutionVariable
-from sagemaker.workflow.parameters import Parameter
-from sagemaker.workflow.properties import Properties
-
-PipelineNonPrimitiveInputTypes = Union[ExecutionVariable, Expression, Parameter, Properties]
 
 
 def is_pipeline_variable(var: object) -> bool: