Merge pull request #1323 from cmu-delphi/outlier-warnings

Change outlier checks as warnings

Author: krivard

_delphi_utils_python/delphi_utils/validator/dynamic.py

@@ -525,7 +525,7 @@ def outlier_nearby(frame):
 
         if source_outliers.shape[0] > 0:
             for time_val in source_outliers["time_value"].unique():
-                report.add_raised_error(
+                report.add_raised_warning(
                     ValidationFailure(
                         "check_positive_negative_spikes",
                         time_val,
@@ -637,7 +637,7 @@ def check_avg_val_vs_reference(self, df_to_test, df_to_reference, checking_date,
             thres["mean_abs_z"])).any()
 
         if mean_z_high or mean_abs_z_high:
-            report.add_raised_error(
+            report.add_raised_warning(
                 ValidationFailure(
                     "check_test_vs_reference_avg_changed",
                     checking_date,

_delphi_utils_python/delphi_utils/validator/report.py

@@ -103,6 +103,19 @@ def log(self, logger=None):
                 checks_suppressed = self.num_suppressed,
                 warnings = len(self.raised_warnings),
                 phase="validation")
+        # Threshold for slack alerts if warnings are excessive,
+        # Currently extremely strict, set by observation of 1 month's logs
+        excessive_warnings = self.total_checks > 0 and \
+            (len(self.raised_warnings) > 200 or \
+            len(self.raised_warnings) / self.total_checks > 0.015)
+        if excessive_warnings:
+            logger.info("Excessive number of warnings",
+                data_source = self.data_source,
+                checks_run = self.total_checks,
+                checks_failed = len(self.unsuppressed_errors),
+                checks_suppressed = self.num_suppressed,
+                warnings = len(self.raised_warnings),
+                phase = "validation")
         for error in self.unsuppressed_errors:
             date_str = "*" if error.date is None else error.date.isoformat()
             logger.critical(str(error),

_delphi_utils_python/tests/validator/test_dynamic.py

@@ -202,8 +202,8 @@ def test_10x_val(self):
             test_df, ref_df,
             datetime.combine(date.today(), datetime.min.time()), "geo", "signal", report)
 
-        assert len(report.raised_errors) == 1
-        assert report.raised_errors[0].check_name == "check_test_vs_reference_avg_changed"
+        assert len(report.raised_warnings) == 1
+        assert report.raised_warnings[0].check_name == "check_test_vs_reference_avg_changed"
 
     def test_100x_val(self):
         validator = DynamicValidator(self.params)
@@ -223,8 +223,8 @@ def test_100x_val(self):
             test_df, ref_df,
             datetime.combine(date.today(), datetime.min.time()), "geo", "signal", report)
 
-        assert len(report.raised_errors) == 1
-        assert report.raised_errors[0].check_name == "check_test_vs_reference_avg_changed"
+        assert len(report.raised_warnings) == 1
+        assert report.raised_warnings[0].check_name == "check_test_vs_reference_avg_changed"
 
     def test_1000x_val(self):
         validator = DynamicValidator(self.params)
@@ -244,8 +244,8 @@ def test_1000x_val(self):
             test_df, ref_df,
             datetime.combine(date.today(), datetime.min.time()), "geo", "signal", report)
 
-        assert len(report.raised_errors) == 1
-        assert report.raised_errors[0].check_name == "check_test_vs_reference_avg_changed"
+        assert len(report.raised_warnings) == 1
+        assert report.raised_warnings[0].check_name == "check_test_vs_reference_avg_changed"
 
 
 class TestDataOutlier:
@@ -292,8 +292,8 @@ def test_pos_outlier(self):
         validator.check_positive_negative_spikes(
             test_df, ref_df, "state", "signal", report)
 
-        assert len(report.raised_errors) == 2
-        assert report.raised_errors[0].check_name == "check_positive_negative_spikes"
+        assert len(report.raised_warnings) == 2
+        assert report.raised_warnings[0].check_name == "check_positive_negative_spikes"
 
     def test_neg_outlier(self):
         validator = DynamicValidator(self.params)
@@ -329,8 +329,8 @@ def test_neg_outlier(self):
         validator.check_positive_negative_spikes(
             test_df, ref_df, "state", "signal", report)
 
-        assert len(report.raised_errors) == 2
-        assert report.raised_errors[0].check_name == "check_positive_negative_spikes"
+        assert len(report.raised_warnings) == 2
+        assert report.raised_warnings[0].check_name == "check_positive_negative_spikes"
 
     def test_zero_outlier(self):
         validator = DynamicValidator(self.params)
@@ -365,8 +365,8 @@ def test_zero_outlier(self):
         validator.check_positive_negative_spikes(
             test_df, ref_df, "state", "signal", report)
 
-        assert len(report.raised_errors) == 1
-        assert report.raised_errors[0].check_name == "check_positive_negative_spikes"
+        assert len(report.raised_warnings) == 1
+        assert report.raised_warnings[0].check_name == "check_positive_negative_spikes"
 
     def test_no_outlier(self):
         validator = DynamicValidator(self.params)
@@ -402,7 +402,7 @@ def test_no_outlier(self):
         validator.check_positive_negative_spikes(
             test_df, ref_df, "state", "signal", report)
 
-        assert len(report.raised_errors) == 0
+        assert len(report.raised_warnings) == 0
 
     def test_source_api_overlap(self):
         validator = DynamicValidator(self.params)
@@ -438,5 +438,5 @@ def test_source_api_overlap(self):
         validator.check_positive_negative_spikes(
             test_df, ref_df, "state", "signal", report)
 
-        assert len(report.raised_errors) == 2
-        assert report.raised_errors[0].check_name == "check_positive_negative_spikes"
+        assert len(report.raised_warnings) == 2
+        assert report.raised_warnings[0].check_name == "check_positive_negative_spikes"