change: Add Pipeline annotation in model base class and tensorflow estimator

Author: Dewen Qi

src/sagemaker/drift_check_baselines.py

@@ -13,21 +13,25 @@
 """This file contains code related to drift check baselines"""
 from __future__ import absolute_import
 
+from typing import Optional
+
+from sagemaker.model_metrics import MetricsSource, FileSource
+
 
 class DriftCheckBaselines(object):
     """Accepts drift check baselines parameters for conversion to request dict."""
 
     def __init__(
         self,
-        model_statistics=None,
-        model_constraints=None,
-        model_data_statistics=None,
-        model_data_constraints=None,
-        bias_config_file=None,
-        bias_pre_training_constraints=None,
-        bias_post_training_constraints=None,
-        explainability_constraints=None,
-        explainability_config_file=None,
+        model_statistics: Optional[MetricsSource] = None,
+        model_constraints: Optional[MetricsSource] = None,
+        model_data_statistics: Optional[MetricsSource] = None,
+        model_data_constraints: Optional[MetricsSource] = None,
+        bias_config_file: Optional[FileSource] = None,
+        bias_pre_training_constraints: Optional[MetricsSource] = None,
+        bias_post_training_constraints: Optional[MetricsSource] = None,
+        explainability_constraints: Optional[MetricsSource] = None,
+        explainability_config_file: Optional[FileSource] = None,
     ):
         """Initialize a ``DriftCheckBaselines`` instance and turn parameters into dict.
 

src/sagemaker/metadata_properties.py

@@ -13,16 +13,20 @@
 """This file contains code related to metadata properties."""
 from __future__ import absolute_import
 
+from typing import Optional, Union
+
+from sagemaker.workflow.entities import PipelineVariable
+
 
 class MetadataProperties(object):
     """Accepts metadata properties parameters for conversion to request dict."""
 
     def __init__(
         self,
-        commit_id=None,
-        repository=None,
-        generated_by=None,
-        project_id=None,
+        commit_id: Optional[Union[str, PipelineVariable]] = None,
+        repository: Optional[Union[str, PipelineVariable]] = None,
+        generated_by: Optional[Union[str, PipelineVariable]] = None,
+        project_id: Optional[Union[str, PipelineVariable]] = None,
     ):
         """Initialize a ``MetadataProperties`` instance and turn parameters into dict.
 

src/sagemaker/model.py

@@ -19,7 +19,7 @@
 import os
 import re
 import copy
-from typing import List, Dict
+from typing import List, Dict, Optional, Union
 
 import sagemaker
 from sagemaker import (
@@ -30,15 +30,20 @@
     utils,
     git_utils,
 )
+from sagemaker.session import Session
+from sagemaker.model_metrics import ModelMetrics
 from sagemaker.deprecations import removed_kwargs
+from sagemaker.drift_check_baselines import DriftCheckBaselines
+from sagemaker.metadata_properties import MetadataProperties
 from sagemaker.predictor import PredictorBase
 from sagemaker.serverless import ServerlessInferenceConfig
 from sagemaker.transformer import Transformer
 from sagemaker.jumpstart.utils import add_jumpstart_tags, get_jumpstart_base_name_if_jumpstart_model
-from sagemaker.utils import unique_name_from_base
+from sagemaker.utils import unique_name_from_base, to_string
 from sagemaker.async_inference import AsyncInferenceConfig
 from sagemaker.predictor_async import AsyncPredictor
 from sagemaker.workflow import is_pipeline_variable
+from sagemaker.workflow.entities import PipelineVariable
 from sagemaker.workflow.pipeline_context import runnable_by_pipeline, PipelineSession
 
 LOGGER = logging.getLogger("sagemaker")
@@ -78,23 +83,23 @@ class Model(ModelBase):
 
     def __init__(
         self,
-        image_uri,
-        model_data=None,
-        role=None,
-        predictor_cls=None,
-        env=None,
-        name=None,
-        vpc_config=None,
-        sagemaker_session=None,
-        enable_network_isolation=False,
-        model_kms_key=None,
-        image_config=None,
-        source_dir=None,
-        code_location=None,
-        entry_point=None,
-        container_log_level=logging.INFO,
-        dependencies=None,
-        git_config=None,
+        image_uri: Union[str, PipelineVariable],
+        model_data: Optional[Union[str, PipelineVariable]] = None,
+        role: Optional[str] = None,
+        predictor_cls: Optional[callable] = None,
+        env: Optional[Dict[str, Union[str, PipelineVariable]]] = None,
+        name: Optional[str] = None,
+        vpc_config: Optional[Dict[str, List[Union[str, PipelineVariable]]]] = None,
+        sagemaker_session: Optional[Session] = None,
+        enable_network_isolation: Union[bool, PipelineVariable] = False,
+        model_kms_key: Optional[str] = None,
+        image_config: Optional[Dict[str, Union[str, PipelineVariable]]] = None,
+        source_dir: Optional[str] = None,
+        code_location: Optional[str] = None,
+        entry_point: Optional[str] = None,
+        container_log_level: Union[int, PipelineVariable] = logging.INFO,
+        dependencies: Optional[List[str]] = None,
+        git_config: Optional[Dict[str, str]] = None,
     ):
         """Initialize an SageMaker ``Model``.
 
@@ -294,22 +299,22 @@ def __init__(
     @runnable_by_pipeline
     def register(
         self,
-        content_types,
-        response_types,
-        inference_instances=None,
-        transform_instances=None,
-        model_package_name=None,
-        model_package_group_name=None,
-        image_uri=None,
-        model_metrics=None,
-        metadata_properties=None,
-        marketplace_cert=False,
-        approval_status=None,
-        description=None,
-        drift_check_baselines=None,
-        customer_metadata_properties=None,
-        validation_specification=None,
-        domain=None,
+        content_types: List[Union[str, PipelineVariable]],
+        response_types: List[Union[str, PipelineVariable]],
+        inference_instances: Optional[List[Union[str, PipelineVariable]]] = None,
+        transform_instances: Optional[List[Union[str, PipelineVariable]]] = None,
+        model_package_name: Optional[Union[str, PipelineVariable]] = None,
+        model_package_group_name: Optional[Union[str, PipelineVariable]] = None,
+        image_uri: Optional[Union[str, PipelineVariable]] = None,
+        model_metrics: Optional[ModelMetrics] = None,
+        metadata_properties: Optional[MetadataProperties] = None,
+        marketplace_cert: bool = False,
+        approval_status: Optional[Union[str, PipelineVariable]] = None,
+        description: Optional[str] = None,
+        drift_check_baselines: Optional[DriftCheckBaselines] = None,
+        customer_metadata_properties: Optional[Dict[str, Union[str, PipelineVariable]]] = None,
+        validation_specification: Optional[Union[str, PipelineVariable]] = None,
+        domain: Optional[Union[str, PipelineVariable]] = None,
     ):
         """Creates a model package for creating SageMaker models or listing on Marketplace.
 
@@ -385,10 +390,10 @@ def register(
     @runnable_by_pipeline
     def create(
         self,
-        instance_type: str = None,
-        accelerator_type: str = None,
-        serverless_inference_config: ServerlessInferenceConfig = None,
-        tags: List[Dict[str, str]] = None,
+        instance_type: Optional[str] = None,
+        accelerator_type: Optional[str] = None,
+        serverless_inference_config: Optional[ServerlessInferenceConfig] = None,
+        tags: Optional[List[Dict[str, Union[str, PipelineVariable]]]] = None,
     ):
         """Create a SageMaker Model Entity
 
@@ -570,7 +575,7 @@ def _script_mode_env_vars(self):
         return {
             SCRIPT_PARAM_NAME.upper(): script_name or str(),
             DIR_PARAM_NAME.upper(): dir_name or str(),
-            CONTAINER_LOG_LEVEL_PARAM_NAME.upper(): str(self.container_log_level),
+            CONTAINER_LOG_LEVEL_PARAM_NAME.upper(): to_string(self.container_log_level),
             SAGEMAKER_REGION_PARAM_NAME.upper(): self.sagemaker_session.boto_region_name,
         }
 
@@ -1239,19 +1244,19 @@ class FrameworkModel(Model):
 
     def __init__(
         self,
-        model_data,
-        image_uri,
-        role,
-        entry_point,
-        source_dir=None,
-        predictor_cls=None,
-        env=None,
-        name=None,
-        container_log_level=logging.INFO,
-        code_location=None,
-        sagemaker_session=None,
-        dependencies=None,
-        git_config=None,
+        model_data: Union[str, PipelineVariable],
+        image_uri: Union[str, PipelineVariable],
+        role: str,
+        entry_point: str,
+        source_dir: Optional[str] = None,
+        predictor_cls: Optional[callable] = None,
+        env: Optional[Dict[str, Union[str, PipelineVariable]]] = None,
+        name: Optional[str] = None,
+        container_log_level: int = logging.INFO,
+        code_location: Optional[str] = None,
+        sagemaker_session: Optional[Session] = None,
+        dependencies: Optional[List[str]] = None,
+        git_config: Optional[Dict[str, str]] = None,
         **kwargs,
     ):
         """Initialize a ``FrameworkModel``.

src/sagemaker/model_metrics.py

@@ -13,20 +13,24 @@
 """This file contains code related to model metrics, including metric source and file source."""
 from __future__ import absolute_import
 
+from typing import Optional, Union
+
+from sagemaker.workflow.entities import PipelineVariable
+
 
 class ModelMetrics(object):
     """Accepts model metrics parameters for conversion to request dict."""
 
     def __init__(
         self,
-        model_statistics=None,
-        model_constraints=None,
-        model_data_statistics=None,
-        model_data_constraints=None,
-        bias=None,
-        explainability=None,
-        bias_pre_training=None,
-        bias_post_training=None,
+        model_statistics: Optional["MetricsSource"] = None,
+        model_constraints: Optional["MetricsSource"] = None,
+        model_data_statistics: Optional["MetricsSource"] = None,
+        model_data_constraints: Optional["MetricsSource"] = None,
+        bias: Optional["MetricsSource"] = None,
+        explainability: Optional["MetricsSource"] = None,
+        bias_pre_training: Optional["MetricsSource"] = None,
+        bias_post_training: Optional["MetricsSource"] = None,
     ):
         """Initialize a ``ModelMetrics`` instance and turn parameters into dict.
 
@@ -99,9 +103,9 @@ class MetricsSource(object):
 
     def __init__(
         self,
-        content_type,
-        s3_uri,
-        content_digest=None,
+        content_type: Union[str, PipelineVariable],
+        s3_uri: Union[str, PipelineVariable],
+        content_digest: Optional[Union[str, PipelineVariable]] = None,
     ):
         """Initialize a ``MetricsSource`` instance and turn parameters into dict.
 
@@ -127,9 +131,9 @@ class FileSource(object):
 
     def __init__(
         self,
-        s3_uri,
-        content_digest=None,
-        content_type=None,
+        s3_uri: Union[str, PipelineVariable],
+        content_digest: Optional[Union[str, PipelineVariable]] = None,
+        content_type: Optional[Union[str, PipelineVariable]] = None,
     ):
         """Initialize a ``FileSource`` instance and turn parameters into dict.
 

src/sagemaker/serverless/serverless_inference_config.py

@@ -27,8 +27,8 @@ class ServerlessInferenceConfig(object):
 
     def __init__(
         self,
-        memory_size_in_mb=2048,
-        max_concurrency=5,
+        memory_size_in_mb: int = 2048,
+        max_concurrency: int = 5,
     ):
         """Initialize a ServerlessInferenceConfig object for serverless inference configuration.
 

src/sagemaker/session.py

@@ -2633,7 +2633,9 @@ def _create_model_request(
             request["VpcConfig"] = vpc_config
 
         if enable_network_isolation:
-            request["EnableNetworkIsolation"] = True
+            # enable_network_isolation may be a pipeline variable which is
+            # parsed in execution time
+            request["EnableNetworkIsolation"] = enable_network_isolation
 
         return request
 

src/sagemaker/tensorflow/estimator.py

@@ -14,6 +14,7 @@
 from __future__ import absolute_import
 
 import logging
+from typing import Optional, Union, Dict
 
 from packaging import version
 
@@ -27,6 +28,7 @@
 from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
 from sagemaker.workflow import is_pipeline_variable
 from sagemaker.tensorflow.training_compiler.config import TrainingCompilerConfig
+from sagemaker.workflow.entities import PipelineVariable
 
 logger = logging.getLogger("sagemaker")
 
@@ -41,12 +43,12 @@ class TensorFlow(Framework):
 
     def __init__(
         self,
-        py_version=None,
-        framework_version=None,
-        model_dir=None,
-        image_uri=None,
-        distribution=None,
-        compiler_config=None,
+        py_version: Optional[str] = None,
+        framework_version: Optional[str] = None,
+        model_dir: Optional[Union[str, PipelineVariable]] = None,
+        image_uri: Optional[Union[str, PipelineVariable]] = None,
+        distribution: Optional[Dict[str, str]] = None,
+        compiler_config: Optional[TrainingCompilerConfig] = None,
         **kwargs,
     ):
         """Initialize a ``TensorFlow`` estimator.
@@ -183,7 +185,7 @@ def __init__(
         self.py_version = py_version
         self.instance_type = instance_type
 
-        if distribution is not None:
+        if distribution is not None and instance_type is not None:
             fw.warn_if_parameter_server_with_multi_gpu(
                 training_instance_type=instance_type, distribution=distribution
             )
@@ -254,6 +256,8 @@ def _only_legacy_mode_supported(self):
 
     def _only_python_3_supported(self):
         """Placeholder docstring"""
+        if not self.framework_version:
+            return False
         return version.Version(self.framework_version) > self._HIGHEST_PYTHON_2_VERSION
 
     @classmethod

src/sagemaker/utils.py

@@ -33,7 +33,7 @@
 
 from sagemaker import deprecations
 from sagemaker.session_settings import SessionSettings
-
+from sagemaker.workflow import is_pipeline_variable
 
 ECR_URI_PATTERN = r"^(\d+)(\.)dkr(\.)ecr(\.)(.+)(\.)(.*)(/)(.*:.*)$"
 MAX_BUCKET_PATHS_COUNT = 5
@@ -722,3 +722,14 @@ def get_data_bucket(self, region_requested=None):
 
 
 get_ecr_image_uri_prefix = deprecations.removed_function("get_ecr_image_uri_prefix")
+
+
+def to_string(obj: object):
+    """Convert an object to string
+
+    This helper function handles converting PipelineVariable object to string as well
+
+    Args:
+        obj (object): The object to be converted
+    """
+    return obj.to_string() if is_pipeline_variable(obj) else str(obj)