ensure smoothed num <= smoothed den (for synthetic data), deescalate assert to warning, remove extra print

Author: mariajahja

emr_hosp/delphi_emr_hosp/sensor.py

@@ -105,6 +105,12 @@ def fit(y_data, sensor_dates, geo_id):
         # the left_gauss_linear smoother is not guaranteed to return values greater than 0
         smoothed_total_counts = np.clip(left_gauss_linear(total_counts.values), 0, None)
         smoothed_total_visits = np.clip(left_gauss_linear(total_visits.values), 0, None)
+
+        # in smoothing, the numerator may have become more than the denominator
+        # simple fix is to clip the max values elementwise to the denominator (note that
+        # this has only been observed in synthetic data)
+        smoothed_total_counts = np.clip(smoothed_total_counts, 0, smoothed_total_visits)
+
         smoothed_total_rates = (
                 (smoothed_total_counts + 0.5) / (smoothed_total_visits + 1)
         )

emr_hosp/delphi_emr_hosp/update_sensor.py

@@ -56,7 +56,8 @@ def write_to_csv(output_dict, out_name, output_path="."):
                 if all_include[geo_id][i]:
                     assert not np.isnan(sensor), "value for included sensor is nan"
                     assert not np.isnan(se), "se for included sensor is nan"
-                    assert sensor < 90, f"value suspiciously high, {geo_id}: {sensor}"
+                    if sensor > 90:
+                        logging.warning(f"value suspiciously high, {geo_id}: {sensor}")
                     assert se < 5, f"se suspiciously high, {geo_id}: {se}"
 
                     # for privacy reasons we will not report the standard error
@@ -136,7 +137,6 @@ def update_sensor(
     unique_geo_ids = list(sorted(np.unique(data_frame.index.get_level_values(0))))
 
     # for each location, fill in all missing dates with 0 values
-    print(len(unique_geo_ids), len(fit_dates))
     multiindex = pd.MultiIndex.from_product((unique_geo_ids, fit_dates),
                                             names=[geo, "date"])
     assert (len(multiindex) <= (Constants.MAX_GEO[geo] * len(fit_dates))

emr_hosp/tests/test_update_sensor.py

@@ -118,16 +118,10 @@ def test_write_to_csv_wrong_results(self):
         with pytest.raises(AssertionError):
             write_to_csv(res2, "name_of_signal", td.name)
 
-        # large sensor value
+        # large se value
         res3 = deepcopy(res0)
-        res3["rates"]["a"][0] = 95
+        res3["se"]["a"][0] = 10
         with pytest.raises(AssertionError):
             write_to_csv(res3, "name_of_signal", td.name)
 
-        # large se value
-        res4 = deepcopy(res0)
-        res4["se"]["a"][0] = 10
-        with pytest.raises(AssertionError):
-            write_to_csv(res4, "name_of_signal", td.name)
-
         td.cleanup()