feature: Add business details and hyper parameters fields and update test_model_card.py (aws#3639)

Co-authored-by: Haonian Wang <[email protected]>

Author: 2 people

doc/api/governance/model_card.rst

@@ -41,3 +41,9 @@ see `Amazon SageMaker Model Cards <https://docs.aws.amazon.com/sagemaker/latest/
 
 .. autoclass:: TrainingJobDetails
     :show-inheritance:
+
+.. autoclass:: BusinessDetails
+    :show-inheritance:
+
+.. autoclass:: HyperParameter
+    :show-inheritance:

src/sagemaker/model_card/__init__.py

@@ -16,8 +16,10 @@
     Environment,
     ModelOverview,
     IntendedUses,
+    BusinessDetails,
     ObjectiveFunction,
     TrainingMetric,
+    HyperParameter,
     Metric,
     Function,
     TrainingJobDetails,

src/sagemaker/model_card/model_card.py

@@ -34,6 +34,8 @@
     TRAINING_DATASETS_MAX_SIZE,
     TRAINING_METRICS_MAX_SIZE,
     USER_PROVIDED_TRAINING_METRICS_MAX_SIZE,
+    HYPER_PARAMETERS_MAX_SIZE,
+    USER_PROVIDED_HYPER_PARAMETERS_MAX_SIZE,
     EVALUATION_DATASETS_MAX_SIZE,
 )
 from sagemaker.model_card.helpers import (
@@ -235,6 +237,27 @@ def __init__(
         self.explanations_for_risk_rating = explanations_for_risk_rating
 
 
+class BusinessDetails(_DefaultToRequestDict, _DefaultFromDict):
+    """The business details of a model."""
+
+    def __init__(
+        self,
+        business_problem: Optional[str] = None,
+        business_stakeholders: Optional[str] = None,
+        line_of_business: Optional[str] = None,
+    ):
+        """Initialize an Business Details object.
+
+        Args:
+            business_problem (str, optional): The business problem of this model (default: None).
+            business_stakeholders (str, optional): The business stakeholders for this model (default: None).
+            line_of_business (str, optional): The line of business for this model (default: None).
+        """  # noqa E501 # pylint: disable=line-too-long
+        self.business_problem = business_problem
+        self.business_stakeholders = business_stakeholders
+        self.line_of_business = line_of_business
+
+
 class Function(_DefaultToRequestDict, _DefaultFromDict):
     """Function details."""
 
@@ -363,6 +386,24 @@ def __init__(
         self.notes = notes
 
 
+class HyperParameter(_DefaultToRequestDict, _DefaultFromDict):
+    """Hyper-Parameters data."""
+
+    def __init__(
+        self,
+        name: str,
+        value: str,
+    ):
+        """Initialize a HyperParameter object.
+
+        Args:
+            name (str): The hyper parameter name.
+            value (str): The hyper parameter value.
+        """
+        self.name = name
+        self.value = value
+
+
 class TrainingJobDetails(_DefaultToRequestDict, _DefaultFromDict):
     """The overview of a training job."""
 
@@ -371,6 +412,10 @@ class TrainingJobDetails(_DefaultToRequestDict, _DefaultFromDict):
     user_provided_training_metrics = _IsList(
         TrainingMetric, USER_PROVIDED_TRAINING_METRICS_MAX_SIZE
     )
+    hyper_parameters = _IsList(HyperParameter, HYPER_PARAMETERS_MAX_SIZE)
+    user_provided_hyper_parameters = _IsList(
+        HyperParameter, USER_PROVIDED_HYPER_PARAMETERS_MAX_SIZE
+    )
     training_environment = _IsModelCardObject(Environment)
 
     def __init__(
@@ -380,6 +425,8 @@ def __init__(
         training_environment: Optional[Environment] = None,
         training_metrics: Optional[List[TrainingMetric]] = None,
         user_provided_training_metrics: Optional[List[TrainingMetric]] = None,
+        hyper_parameters: Optional[List[HyperParameter]] = None,
+        user_provided_hyper_parameters: Optional[List[HyperParameter]] = None,
     ):
         """Initialize a Training Job Details object.
 
@@ -389,12 +436,16 @@ def __init__(
             training_environment (Environment, optional): The SageMaker training image URI. (default: None).
             training_metrics (list[TrainingMetric], optional): SageMaker training job results. The maximum `training_metrics` list length is 50 (default: None).
             user_provided_training_metrics (list[TrainingMetric], optional): Custom training job results. The maximum `user_provided_training_metrics` list length is 50 (default: None).
+            hyper_parameters (list[HyperParameter], optional): SageMaker hyper parameter results. The maximum `hyper_parameters` list length is 100 (default: None).
+            user_provided_hyper_parameters (list[HyperParameter], optional): Custom hyper parameter results. The maximum `user_provided_hyper_parameters` list length is 100 (default: None).
         """  # noqa E501 # pylint: disable=line-too-long
         self.training_arn = training_arn
         self.training_datasets = training_datasets
         self.training_environment = training_environment
         self.training_metrics = training_metrics
         self.user_provided_training_metrics = user_provided_training_metrics
+        self.hyper_parameters = hyper_parameters
+        self.user_provided_hyper_parameters = user_provided_hyper_parameters
 
 
 class TrainingDetails(_DefaultToRequestDict, _DefaultFromDict):
@@ -442,6 +493,10 @@ def _create_training_details(training_job_data: dict, cls: "TrainingDetails", **
                 ]
                 if "FinalMetricDataList" in training_job_data
                 else [],
+                "hyper_parameters": [
+                    HyperParameter(key, value)
+                    for key, value in training_job_data["HyperParameters"].items()
+                ],
             }
             kwargs.update({"training_job_details": TrainingJobDetails(**job)})
             instance = cls(**kwargs)
@@ -568,6 +623,16 @@ def add_metric(self, metric: TrainingMetric):
             self.training_job_details = TrainingJobDetails()
         self.training_job_details.user_provided_training_metrics.append(metric)
 
+    def add_parameter(self, parameter: HyperParameter):
+        """Add custom hyper-parameter.
+
+        Args:
+            parameter (HyperParameter): The custom parameter to add.
+        """
+        if not self.training_job_details:
+            self.training_job_details = TrainingJobDetails()
+        self.training_job_details.user_provided_hyper_parameters.append(parameter)
+
 
 class MetricGroup(_DefaultToRequestDict, _DefaultFromDict):
     """Group of metric data"""
@@ -777,6 +842,7 @@ class ModelCard(object):
     status = _OneOf(ModelCardStatusEnum)
     model_overview = _IsModelCardObject(ModelOverview)
     intended_uses = _IsModelCardObject(IntendedUses)
+    business_details = _IsModelCardObject(BusinessDetails)
     training_details = _IsModelCardObject(TrainingDetails)
     evaluation_details = _IsList(EvaluationJob)
     additional_information = _IsModelCardObject(AdditionalInformation)
@@ -793,6 +859,7 @@ def __init__(
         last_modified_by: Optional[dict] = None,
         model_overview: Optional[ModelOverview] = None,
         intended_uses: Optional[IntendedUses] = None,
+        business_details: Optional[BusinessDetails] = None,
         training_details: Optional[TrainingDetails] = None,
         evaluation_details: Optional[List[EvaluationJob]] = None,
         additional_information: Optional[AdditionalInformation] = None,
@@ -811,6 +878,7 @@ def __init__(
             last_modified_by (dict, optional): The group or individual that last modified the model card (default: None).
             model_overview (ModelOverview, optional): An overview of the model (default: None).
             intended_uses (IntendedUses, optional): The intended uses of the model (default: None).
+            business_details (BusinessDetails, optional): The business details of the model (default: None).
             training_details (TrainingDetails, optional): The training details of the model (default: None).
             evaluation_details (List[EvaluationJob], optional): The evaluation details of the model (default: None).
             additional_information (AdditionalInformation, optional): Additional information about the model (default: None).
@@ -826,6 +894,7 @@ def __init__(
         self.last_modified_by = last_modified_by
         self.model_overview = model_overview
         self.intended_uses = intended_uses
+        self.business_details = business_details
         self.training_details = training_details
         self.evaluation_details = evaluation_details
         self.additional_information = additional_information

src/sagemaker/model_card/schema_constraints.py

@@ -84,4 +84,6 @@ class MetricTypeEnum(str, Enum):
 TRAINING_DATASETS_MAX_SIZE = 15
 TRAINING_METRICS_MAX_SIZE = 50
 USER_PROVIDED_TRAINING_METRICS_MAX_SIZE = 50
+HYPER_PARAMETERS_MAX_SIZE = 100
+USER_PROVIDED_HYPER_PARAMETERS_MAX_SIZE = 100
 EVALUATION_DATASETS_MAX_SIZE = 10

tests/unit/test_model_card.py

@@ -27,8 +27,10 @@
     Environment,
     ModelOverview,
     IntendedUses,
+    BusinessDetails,
     ObjectiveFunction,
     TrainingMetric,
+    HyperParameter,
     Metric,
     TrainingDetails,
     MetricGroup,
@@ -75,6 +77,11 @@
 RISK_RATING = schema_constraints.RiskRatingEnum.LOW
 EXPLANATIONS_FOR_RISK_RATING = "ramdomly the first example"
 
+# business details auguments
+BUSINESS_PROBLEM = "mock model for business problem testing"
+BUSINESS_STAKEHOLDERS = "business stakeholders testing"
+LINE_OF_BUSINESS = "how many business models"
+
 # training details arguments
 OBJECITVE_FUNCTION_FUNC = schema_constraints.ObjectiveFunctionEnum.MINIMIZE
 OBJECTIVE_FUNCTION_FACET = schema_constraints.FacetEnum.LOSS
@@ -89,6 +96,10 @@
 USER_METRIC_NAME = "test_metric"
 USER_METRIC = TrainingMetric(name=USER_METRIC_NAME, value=1)
 USER_PROVIDED_TRAINING_METRICS = [USER_METRIC]
+HYPER_PARAMETER = [HyperParameter(name="binary_f_beta", value=0.965)]
+USER_PARAMETER_NAME = "test_parameter"
+USER_PARAMETER = HyperParameter(name=USER_PARAMETER_NAME, value=1)
+USER_PROVIDED_HYPER_PARAMETER = [USER_PARAMETER]
 
 # evaluation job arguments
 EVALUATION_JOB_NAME = "evaluation job 1"
@@ -350,6 +361,22 @@
                         "Timestamp": datetime.datetime(2022, 9, 5, 19, 18, 40),
                     },
                 ],
+                "HyperParameters": {
+                    "_kfold": "5",
+                    "_tuning_objective_metric": "validation:accuracy",
+                    "alpha": "0.0037170512924477993",
+                    "colsample_bytree": "0.7476726040667319",
+                    "eta": "0.011391935592233605",
+                    "eval_metric": "accuracy,f1,balanced_accuracy,precision_macro,recall_macro,mlogloss",
+                    "gamma": "1.8903517751689445",
+                    "lambda": "0.5098604662224621",
+                    "max_depth": "3",
+                    "min_child_weight": "5.081388147234708e-06",
+                    "num_class": "28",
+                    "num_round": "165",
+                    "objective": "multi:softprob",
+                    "subsample": "0.8828549481113146",
+                },
                 "CreatedBy": {},
             }
         }
@@ -583,6 +610,17 @@ def fixture_fixture_intended_uses_example():
     return test_example
 
 
+@pytest.fixture(name="business_details_example")
+def fixture_fixture_business_details_example():
+    """Example business details instance."""
+    test_example = BusinessDetails(
+        business_problem=BUSINESS_PROBLEM,
+        business_stakeholders=BUSINESS_STAKEHOLDERS,
+        line_of_business=LINE_OF_BUSINESS,
+    )
+    return test_example
+
+
 @pytest.fixture(name="training_details_example")
 def fixture_fixture_training_details_example():
     """Example training details instance."""
@@ -601,6 +639,7 @@ def fixture_fixture_training_details_example():
             training_datasets=TRAINING_DATASETS,
             training_environment=TRAINING_ENVIRONMENT,
             training_metrics=TRAINING_METRICS,
+            hyper_parameters=HYPER_PARAMETER,
         ),
     )
     return test_example
@@ -637,6 +676,7 @@ def test_create_model_card(
     session,
     model_overview_example,
     intended_uses_example,
+    business_details_example,
     training_details_example,
     evaluation_details_example,
     additional_information_example,
@@ -649,6 +689,7 @@ def test_create_model_card(
         status=MODEL_CARD_STATUS,
         model_overview=model_overview_example,
         intended_uses=intended_uses_example,
+        business_details=business_details_example,
         training_details=training_details_example,
         evaluation_details=evaluation_details_example,
         additional_information=additional_information_example,
@@ -1017,6 +1058,9 @@ def test_training_details_autodiscovery_from_model_overview(
     assert len(training_details.training_job_details.training_metrics) == len(
         SEARCH_TRAINING_JOB_EXAMPLE["Results"][0]["TrainingJob"]["FinalMetricDataList"]
     )
+    assert len(training_details.training_job_details.hyper_parameters) == len(
+        SEARCH_TRAINING_JOB_EXAMPLE["Results"][0]["TrainingJob"]["HyperParameters"]
+    )
     assert training_details.training_job_details.training_environment.container_image == [
         TRAINING_IMAGE
     ]
@@ -1046,7 +1090,10 @@ def test_training_details_autodiscovery_from_model_overview_autopilot(
         model_overview=model_overview_example, sagemaker_session=session
     )
 
+    # MetricDefinitions is empty
     assert len(training_details.training_job_details.training_metrics) == 0
+    # HyperParameters have 3 keys
+    assert len(training_details.training_job_details.hyper_parameters) == 3
 
 
 @patch("sagemaker.Session")
@@ -1063,6 +1110,9 @@ def test_training_details_autodiscovery_from_job_name(session):
     assert len(training_details.training_job_details.training_metrics) == len(
         SEARCH_TRAINING_JOB_EXAMPLE["Results"][0]["TrainingJob"]["FinalMetricDataList"]
     )
+    assert len(training_details.training_job_details.hyper_parameters) == len(
+        SEARCH_TRAINING_JOB_EXAMPLE["Results"][0]["TrainingJob"]["HyperParameters"]
+    )
     assert training_details.training_job_details.training_environment.container_image == [
         TRAINING_IMAGE
     ]
@@ -1091,6 +1141,16 @@ def test_add_user_provided_training_metrics(training_details_example):
     )
 
 
+def test_add_user_provided_hyper_parameters(training_details_example):
+    assert len(training_details_example.training_job_details.user_provided_hyper_parameters) == 0
+    training_details_example.add_parameter(USER_PARAMETER)
+    assert len(training_details_example.training_job_details.user_provided_hyper_parameters) == 1
+    assert (
+        training_details_example.training_job_details.user_provided_hyper_parameters[0].name
+        == USER_PARAMETER_NAME
+    )
+
+
 def test_add_evaluation_metrics_manually():
     evaluation_job = EvaluationJob(name=EVALUATION_JOB_NAME)