Merge branch 'master' into fix-cloudwatch-logs-local

Author: shreyapandit

doc/workflows/pipelines/index.rst

@@ -1,8 +1,8 @@
-################
-Amazon Pipelines
-################
+###################
+SageMaker Pipelines
+###################
 
-SageMaker APIs for creating and managing Amazon Pipelines.
+SageMaker APIs for creating and managing SageMaker Pipelines.
 
 .. toctree::
     :maxdepth: 2

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -33,8 +33,6 @@ Conditions
 
 .. autoclass:: sagemaker.workflow.conditions.ConditionOr
 
-.. autofunction:: sagemaker.workflow.conditions.primitive_or_expr
-
 Entities
 --------
 
@@ -44,7 +42,7 @@ Entities
 
 .. autoclass:: sagemaker.workflow.entities.Expression
 
-Execution_variables
+Execution Variables
 -------------------
 
 .. autoclass:: sagemaker.workflow.execution_variables.ExecutionVariable
@@ -73,12 +71,17 @@ Pipeline
 --------
 
 .. autoclass:: sagemaker.workflow.pipeline.Pipeline
+    :members:
+
+.. autoclass:: sagemaker.workflow.pipeline._PipelineExecution
+    :members:
 
-.. autofunction:: sagemaker.workflow.pipeline.format_start_parameters
+Pipeline Experiment Config
+--------------------------
 
-.. autofunction:: sagemaker.workflow.pipeline.interpolate
+.. autoclass:: sagemaker.workflow.pipeline_experiment_config.PipelineExperimentConfig
 
-.. autofunction:: sagemaker.workflow.pipeline.update_args
+.. autoclass:: sagemaker.workflow.pipeline_experiment_config.PipelineExperimentConfigProperty
 
 Properties
 ----------
@@ -121,7 +124,4 @@ Steps
 
 .. autoclass:: sagemaker.workflow.callback_step.CallbackStep
 
-Utilities
----------
-
-.. autofunction:: sagemaker.workflow.utilities.list_to_request
+.. autoclass:: sagemaker.workflow.steps.CacheConfig

src/sagemaker/clarify.py

@@ -305,10 +305,11 @@ def __init__(
         """Initializes config for SHAP.
 
         Args:
-            baseline (str or list): A list of rows (at least one) or S3 object URI to be used as
-                the baseline dataset in the Kernel SHAP algorithm. The format should be the same
-                as the dataset format. Each row should contain only the feature columns/values
-                and omit the label column/values.
+            baseline (None or str or list): None or S3 object Uri or A list of rows (at least one)
+                to be used asthe baseline dataset in the Kernel SHAP algorithm. The format should
+                be the same as the dataset format. Each row should contain only the feature
+                columns/values and omit the label column/values. If None a baseline will be
+                calculated automatically by using K-means or K-prototypes in the input dataset.
             num_samples (int): Number of samples to be used in the Kernel SHAP algorithm.
                 This number determines the size of the generated synthetic dataset to compute the
                 SHAP values.

src/sagemaker/estimator.py

@@ -164,7 +164,10 @@ def __init__(
                 file:// urls are used for local mode. For example: 'file://model/'
                 will save to the model folder in the current directory.
             output_kms_key (str): Optional. KMS key ID for encrypting the
-                training output (default: None).
+                training output (default: Your IAM role's KMS key for Amazon S3).
+                If you don't provide a KMS key ID, Amazon SageMaker uses the
+                default KMS key for Amazon S3 of the account linked to your
+                IAM role.
             base_job_name (str): Prefix for training job name when the
                 :meth:`~sagemaker.estimator.EstimatorBase.fit` method launches.
                 If not specified, the estimator generates a default job name

src/sagemaker/workflow/condition_step.py

@@ -55,10 +55,10 @@ def __init__(
             conditions (List[Condition]): A list of `sagemaker.workflow.conditions.Condition`
                 instances.
             if_steps (List[Union[Step, StepCollection]]): A list of `sagemaker.workflow.steps.Step`
-                and `sagemaker.workflow.step_collections.StepCollection` instances that are
+                or `sagemaker.workflow.step_collections.StepCollection` instances that are
                 marked as ready for execution if the list of conditions evaluates to True.
             else_steps (List[Union[Step, StepCollection]]): A list of `sagemaker.workflow.steps.Step`
-                and `sagemaker.workflow.step_collections.StepCollection` instances that are
+                or `sagemaker.workflow.step_collections.StepCollection` instances that are
                 marked as ready for execution if the list of conditions evaluates to False.
         """
         super(ConditionStep, self).__init__(name, StepTypeEnum.CONDITION, depends_on)

src/sagemaker/workflow/conditions.py

@@ -66,13 +66,13 @@ class ConditionComparison(Condition):
     """Generic comparison condition that can be used to derive specific condition comparisons.
 
     Attributes:
-        left (ConditionValueType): The execution variable, parameter, or
-            property to use in the comparison.
+        left (Union[ConditionValueType, PrimitiveType]): The execution variable, parameter,
+            property, or Python primitive value to use in the comparison.
         right (Union[ConditionValueType, PrimitiveType]): The execution variable,
             parameter, property, or Python primitive value to compare to.
     """
 
-    left: ConditionValueType = attr.ib(default=None)
+    left: Union[ConditionValueType, PrimitiveType] = attr.ib(default=None)
     right: Union[ConditionValueType, PrimitiveType] = attr.ib(default=None)
 
     def to_request(self) -> RequestType:
@@ -87,12 +87,16 @@ def to_request(self) -> RequestType:
 class ConditionEquals(ConditionComparison):
     """A condition for equality comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct A condition for equality comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -103,12 +107,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
 class ConditionGreaterThan(ConditionComparison):
     """A condition for greater than comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct an instance of ConditionGreaterThan for greater than comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -119,12 +127,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
 class ConditionGreaterThanOrEqualTo(ConditionComparison):
     """A condition for greater than or equal to comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct of ConditionGreaterThanOrEqualTo for greater than or equal to comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -135,12 +147,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
 class ConditionLessThan(ConditionComparison):
     """A condition for less than comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct an instance of ConditionLessThan for less than comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -151,12 +167,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
 class ConditionLessThanOrEqualTo(ConditionComparison):
     """A condition for less than or equal to comparisons."""
 
-    def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
+    def __init__(
+        self,
+        left: Union[ConditionValueType, PrimitiveType],
+        right: Union[ConditionValueType, PrimitiveType],
+    ):
         """Construct ConditionLessThanOrEqualTo for less than or equal to comparisons.
 
         Args:
-            left (ConditionValueType): The execution variable, parameter,
-                or property to use in the comparison.
+            left (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property, or Python primitive value to use in the comparison.
             right (Union[ConditionValueType, PrimitiveType]): The execution
                 variable, parameter, property, or Python primitive value to compare to.
         """
@@ -168,13 +188,15 @@ class ConditionIn(Condition):
     """A condition to check membership."""
 
     def __init__(
-        self, value: ConditionValueType, in_values: List[Union[ConditionValueType, PrimitiveType]]
+        self,
+        value: Union[ConditionValueType, PrimitiveType],
+        in_values: List[Union[ConditionValueType, PrimitiveType]],
     ):
         """Construct a `ConditionIn` condition to check membership.
 
         Args:
-            value (ConditionValueType): The execution variable,
-                parameter, or property to use for the in comparison.
+            value (Union[ConditionValueType, PrimitiveType]): The execution variable,
+                parameter, property or primitive value to check for membership.
             in_values (List[Union[ConditionValueType, PrimitiveType]]): The list
                 of values to check for membership in.
         """

src/sagemaker/workflow/execution_variables.py

@@ -37,10 +37,7 @@ def expr(self) -> RequestType:
 
 
 class ExecutionVariables:
-    """Enum-like class for all ExecutionVariable instances.
-
-    Considerations to move these as module-level constants should be made.
-    """
+    """All available ExecutionVariable."""
 
     START_DATETIME = ExecutionVariable("StartDateTime")
     CURRENT_DATETIME = ExecutionVariable("CurrentDateTime")

src/sagemaker/workflow/functions.py

@@ -24,9 +24,19 @@
 class Join(Expression):
     """Join together properties.
 
+    Examples:
+    Build a Amazon S3 Uri with bucket name parameter and pipeline execution Id and use it
+    as training input::
+
+        bucket = ParameterString('bucket', default_value='my-bucket')
+
+        TrainingInput(
+            s3_data=Join(on='/', ['s3:/', bucket, ExecutionVariables.PIPELINE_EXECUTION_ID]),
+            content_type="text/csv")
+
     Attributes:
-        values (List[Union[PrimitiveType, Parameter]]): The primitive types
-            and parameters to join.
+        values (List[Union[PrimitiveType, Parameter, Expression]]):
+            The primitive type values, parameters, step properties, expressions to join.
         on_str (str): The string to join the values on (Defaults to "").
     """
 

src/sagemaker/workflow/parameters.py

@@ -54,7 +54,7 @@ class Parameter(Entity):
     Attributes:
         name (str): The name of the parameter.
         parameter_type (ParameterTypeEnum): The type of the parameter.
-        default_value (PrimitiveType): The default Python value of the parameter.
+        default_value (PrimitiveType): The default value of the parameter.
     """
 
     name: str = attr.ib(factory=str)
@@ -143,7 +143,7 @@ def _check_default_value_type(cls, value, python_type):
 
 
 class ParameterString(Parameter, str):
-    """Pipeline string parameter for workflow."""
+    """String parameter for pipelines."""
 
     def __new__(cls, *args, **kwargs):  # pylint: disable=unused-argument
         """Subclass str"""
@@ -155,7 +155,11 @@ def __init__(self, name: str, default_value: str = None, enum_values: List[str]
 
         Args:
             name (str): The name of the parameter.
-            default_value (str): The default Python value of the parameter. Defaults to None.
+            default_value (str): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
+            enum_values (List[str]): Enum values for this parameter.
         """
         super(ParameterString, self).__init__(
             name=name, parameter_type=ParameterTypeEnum.STRING, default_value=default_value
@@ -175,7 +179,7 @@ def to_request(self) -> RequestType:
 
 
 class ParameterInteger(Parameter, int):
-    """Pipeline string parameter for workflow."""
+    """Integer parameter for pipelines."""
 
     def __new__(cls, *args, **kwargs):
         """Subclass int"""
@@ -187,15 +191,18 @@ def __init__(self, name: str, default_value: int = None):
 
         Args:
             name (str): The name of the parameter.
-            default_value (int): The default Python value of the parameter.
+            default_value (int): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
         """
         super(ParameterInteger, self).__init__(
             name=name, parameter_type=ParameterTypeEnum.INTEGER, default_value=default_value
         )
 
 
 class ParameterFloat(Parameter, float):
-    """Pipeline float parameter for workflow."""
+    """Float parameter for pipelines."""
 
     def __new__(cls, *args, **kwargs):
         """Subclass float"""
@@ -207,7 +214,10 @@ def __init__(self, name: str, default_value: float = None):
 
         Args:
             name (str): The name of the parameter.
-            default_value (float): The default Python value of the parameter.
+            default_value (float): The default value of the parameter.
+                The default value could be overridden at start of an execution.
+                If not set or it is set to None, a value must be provided
+                at the start of the execution.
         """
         super(ParameterFloat, self).__init__(
             name=name, parameter_type=ParameterTypeEnum.FLOAT, default_value=default_value