recover eval_metrics for kmeans and rcf

Author: Chuyang Deng

doc/algorithms/randomcutforest.rst

@@ -8,7 +8,7 @@ The Amazon SageMaker Random Cut Forest algorithm.
     :undoc-members:
     :show-inheritance:
     :inherited-members:
-    :exclude-members: image_uri, num_trees, num_samples_per_tree, feature_dim, MINI_BATCH_SIZE
+    :exclude-members: image_uri, num_trees, num_samples_per_tree, eval_metrics, feature_dim, MINI_BATCH_SIZE
 
 
 .. autoclass:: sagemaker.RandomCutForestModel

src/sagemaker/amazon/kmeans.py

@@ -43,6 +43,11 @@ class KMeans(AmazonAlgorithmEstimatorBase):
     )
     epochs = hp("epochs", gt(0), "An integer greater-than 0", int)
     center_factor = hp("extra_center_factor", gt(0), "An integer greater-than 0", int)
+    eval_metrics = hp(
+        name="eval_metrics",
+        validation_message='A comma separated list of "msd" or "ssd"',
+        data_type=list,
+    )
 
     def __init__(
         self,
@@ -58,6 +63,7 @@ def __init__(
         half_life_time_size=None,
         epochs=None,
         center_factor=None,
+        eval_metrics=None,
         **kwargs
     ):
         """A k-means clustering
@@ -124,6 +130,11 @@ def __init__(
             center_factor (int): The algorithm will create
                 ``num_clusters * extra_center_factor`` as it runs and reduce the
                 number of centers to ``k`` when finalizing
+            eval_metrics (list): JSON list of metrics types to be used for
+                reporting the score for the model. Allowed values are "msd"
+                Means Square Error, "ssd": Sum of square distance. If test data
+                is provided, the score shall be reported in terms of all
+                requested metrics.
             **kwargs: base class keyword argument values.
 
         .. tip::
@@ -142,6 +153,7 @@ def __init__(
         self.half_life_time_size = half_life_time_size
         self.epochs = epochs
         self.center_factor = center_factor
+        self.eval_metrics = eval_metrics
 
     def create_model(self, vpc_config_override=VPC_CONFIG_DEFAULT, **kwargs):
         """Return a :class:`~sagemaker.amazon.kmeans.KMeansModel` referencing

src/sagemaker/amazon/randomcutforest.py

@@ -31,6 +31,12 @@ class RandomCutForest(AmazonAlgorithmEstimatorBase):
     repo_version = 1
     MINI_BATCH_SIZE = 1000
 
+    eval_metrics = hp(
+        name="eval_metrics",
+        validation_message='A comma separated list of "accuracy" or "precision_recall_fscore"',
+        data_type=list,
+    )
+
     num_trees = hp("num_trees", (ge(50), le(1000)), "An integer in [50, 1000]", int)
     num_samples_per_tree = hp(
         "num_samples_per_tree", (ge(1), le(2048)), "An integer in [1, 2048]", int
@@ -44,6 +50,7 @@ def __init__(
         instance_type,
         num_samples_per_tree=None,
         num_trees=None,
+        eval_metrics=None,
         **kwargs
     ):
         """RandomCutForest is :class:`Estimator` used for anomaly detection.
@@ -92,6 +99,11 @@ def __init__(
                 build each tree in the forest. The total number of samples drawn
                 from the train dataset is num_trees * num_samples_per_tree.
             num_trees (int): Optional. The number of trees used in the forest.
+            eval_metrics (list): Optional. JSON list of metrics types to be used
+                for reporting the score for the model. Allowed values are
+                "accuracy", "precision_recall_fscore": positive and negative
+                precision, recall, and f1 scores. If test data is provided, the
+                score shall be reported in terms of all requested metrics.
             **kwargs: base class keyword argument values.
 
         .. tip::
@@ -104,6 +116,7 @@ def __init__(
         super(RandomCutForest, self).__init__(role, instance_count, instance_type, **kwargs)
         self.num_samples_per_tree = num_samples_per_tree
         self.num_trees = num_trees
+        self.eval_metrics = eval_metrics
 
     def create_model(self, vpc_config_override=VPC_CONFIG_DEFAULT, **kwargs):
         """Return a :class:`~sagemaker.amazon.RandomCutForestModel` referencing

tests/integ/test_airflow_config.py

@@ -111,6 +111,7 @@ def test_kmeans_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_
         kmeans.half_life_time_size = 1
         kmeans.epochs = 1
         kmeans.center_factor = 1
+        kmeans.eval_metrics = ["ssd", "msd"]
 
         records = kmeans.record_set(datasets.one_p_mnist()[0][:100])
 
@@ -385,6 +386,7 @@ def test_rcf_airflow_config_uploads_data_source_to_s3(sagemaker_session, cpu_ins
             instance_type=cpu_instance_type,
             num_trees=50,
             num_samples_per_tree=20,
+            eval_metrics=["accuracy", "precision_recall_fscore"],
             sagemaker_session=sagemaker_session,
         )
 

tests/integ/test_kmeans.py

@@ -12,6 +12,7 @@
 # language governing permissions and limitations under the License.
 from __future__ import absolute_import
 
+import json
 import time
 
 import pytest
@@ -46,6 +47,7 @@ def test_kmeans(sagemaker_session, cpu_instance_type, training_set):
         kmeans.half_life_time_size = 1
         kmeans.epochs = 1
         kmeans.center_factor = 1
+        kmeans.eval_metrics = ["ssd", "msd"]
 
         assert kmeans.hyperparameters() == dict(
             init_method=kmeans.init_method,
@@ -57,6 +59,7 @@ def test_kmeans(sagemaker_session, cpu_instance_type, training_set):
             epochs=str(kmeans.epochs),
             extra_center_factor=str(kmeans.center_factor),
             k=str(kmeans.k),
+            eval_metrics=json.dumps(kmeans.eval_metrics),
             force_dense="True",
         )
 

tests/integ/test_multidatamodel.py

@@ -439,6 +439,7 @@ def __rcf_training_job(
             instance_type=cpu_instance_type,
             num_trees=num_trees,
             num_samples_per_tree=num_samples_per_tree,
+            eval_metrics=["accuracy", "precision_recall_fscore"],
             sagemaker_session=sagemaker_session,
         )
 

tests/integ/test_randomcutforest.py

@@ -34,6 +34,7 @@ def test_randomcutforest(sagemaker_session, cpu_instance_type):
             instance_type=cpu_instance_type,
             num_trees=50,
             num_samples_per_tree=20,
+            eval_metrics=["accuracy", "precision_recall_fscore"],
             sagemaker_session=sagemaker_session,
         )
 

tests/unit/test_kmeans.py

@@ -92,6 +92,7 @@ def test_all_hyperparameters(sagemaker_session):
         half_life_time_size=0,
         epochs=10,
         center_factor=2,
+        eval_metrics=["msd", "ssd"],
         **ALL_REQ_ARGS
     )
     assert kmeans.hyperparameters() == dict(
@@ -104,6 +105,7 @@ def test_all_hyperparameters(sagemaker_session):
         half_life_time_size="0",
         epochs="10",
         extra_center_factor="2",
+        eval_metrics='["msd", "ssd"]',
         force_dense="True",
     )
 
@@ -129,7 +131,7 @@ def test_required_hyper_parameters_value(sagemaker_session, required_hyper_param
         KMeans(sagemaker_session=sagemaker_session, **test_params)
 
 
-@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])])
+@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", 0)])
 def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value):
     with pytest.raises(TypeError):
         test_params = ALL_REQ_ARGS.copy()

tests/unit/test_linear_learner.py

@@ -216,7 +216,7 @@ def test_num_classes_can_be_string_for_multiclass_classifier(sagemaker_session):
     LinearLearner(sagemaker_session=sagemaker_session, **test_params)
 
 
-@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])])
+@pytest.mark.parametrize("iterable_hyper_parameters, value", [("max_iterations", [0])])
 def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value):
     with pytest.raises(TypeError):
         test_params = ALL_REQ_ARGS.copy()

tests/unit/test_randomcutforest.py

@@ -24,6 +24,7 @@
 INSTANCE_TYPE = "ml.c4.xlarge"
 NUM_SAMPLES_PER_TREE = 20
 NUM_TREES = 50
+EVAL_METRICS = ["accuracy", "precision_recall_fscore"]
 
 COMMON_TRAIN_ARGS = {
     "role": ROLE,
@@ -70,13 +71,15 @@ def test_init_required_positional(sagemaker_session):
         INSTANCE_TYPE,
         NUM_SAMPLES_PER_TREE,
         NUM_TREES,
+        EVAL_METRICS,
         sagemaker_session=sagemaker_session,
     )
     assert randomcutforest.role == ROLE
     assert randomcutforest.instance_count == INSTANCE_COUNT
     assert randomcutforest.instance_type == INSTANCE_TYPE
     assert randomcutforest.num_trees == NUM_TREES
     assert randomcutforest.num_samples_per_tree == NUM_SAMPLES_PER_TREE
+    assert randomcutforest.eval_metrics == EVAL_METRICS
 
 
 def test_init_required_named(sagemaker_session):
@@ -92,10 +95,13 @@ def test_all_hyperparameters(sagemaker_session):
         sagemaker_session=sagemaker_session,
         num_trees=NUM_TREES,
         num_samples_per_tree=NUM_SAMPLES_PER_TREE,
+        eval_metrics=EVAL_METRICS,
         **ALL_REQ_ARGS
     )
     assert randomcutforest.hyperparameters() == dict(
-        num_samples_per_tree=str(NUM_SAMPLES_PER_TREE), num_trees=str(NUM_TREES),
+        num_samples_per_tree=str(NUM_SAMPLES_PER_TREE),
+        num_trees=str(NUM_TREES),
+        eval_metrics='["accuracy", "precision_recall_fscore"]',
     )
 
 
@@ -104,7 +110,7 @@ def test_image(sagemaker_session):
     assert image_uris.retrieve("randomcutforest", REGION) == randomcutforest.train_image()
 
 
-@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", [0])])
+@pytest.mark.parametrize("iterable_hyper_parameters, value", [("eval_metrics", 0)])
 def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value):
     with pytest.raises(TypeError):
         test_params = ALL_REQ_ARGS.copy()