Merge pull request #1136 from cmu-delphi/remove-7dav-cumulative

Remove 7dav from cumulative signals

Author: krivard

combo_cases_and_deaths/delphi_combo_cases_and_deaths/run.py

@@ -322,6 +322,7 @@ def run_module(params):
     variants = [tuple((metric, geo_res)+sensor_signal(metric, sensor, smoother))
                 for (metric, geo_res, sensor, smoother) in
                 product(METRICS, GEO_RESOLUTIONS, SENSORS, SMOOTH_TYPES)]
+    variants = [i for i in variants if not ("7dav" in i[2] and "cumulative" in i[2])]
     params = configure(variants, params)
     logger = get_structured_logger(
         __name__, filename=params["common"].get("log_filename"),

combo_cases_and_deaths/tests/receiving/.gitkeep

@@ -1,13 +1,16 @@
 """Tests for running combo cases and deaths indicator."""
 from datetime import date
 from itertools import product
+import os
 import unittest
 from unittest.mock import patch, call
 import pandas as pd
 import numpy as np
 
 from delphi_combo_cases_and_deaths.run import (
-    extend_raw_date_range, get_updated_dates,
+    run_module,
+    extend_raw_date_range,
+    get_updated_dates,
     sensor_signal,
     combine_usafacts_and_jhu,
     compute_special_geo_dfs,
@@ -244,6 +247,50 @@ def test_no_nation_jhu(mock_covidcast_signal):
                       "sample_size": [None]},)
     )
 
+@patch("delphi_combo_cases_and_deaths.run.combine_usafacts_and_jhu")
+def test_output_files(mock_combine):
+    params = {
+        "common": {
+            "export_dir": "./receiving"
+        },
+        "indicator": {
+            "export_start_date": [2020, 4, 1],
+            "source":"indicator-combination",
+            "wip_signal": ""
+        }
+    }
+    mock_combine.return_value = pd.DataFrame(
+            {
+                "geo_id": ["01000"],
+                "val": [10],
+                "timestamp": [pd.to_datetime("2021-01-04")],
+                "issue": [pd.to_datetime("2021-01-04")],
+                "se": 0,
+                "sample_size": 0
+            },
+            index=[1]
+        )
+    run_module(params)
+    csv_files = [f for f in os.listdir("receiving") if f.endswith(".csv")]
+    dates = ["20210104"]
+    geos = ["county", "hrr", "msa", "state", "hhs", "nation"]
+
+    # enumerate metric names.
+    metrics = []
+    for event, span, stat in product(["deaths", "confirmed"],
+                                     ["cumulative", "incidence"],
+                                     ["num", "prop"]):
+        metrics.append("_".join([event, span, stat]))
+        metrics.append("_".join([event, "7dav", span, stat]))
+
+    expected_files = []
+    for date in dates:
+        for geo in geos:
+            for metric in metrics:
+                if "7dav" in metric and "cumulative" in metric:
+                    continue
+                expected_files += [date + "_" + geo + "_" + metric + ".csv"]
+    assert set(csv_files) == set(expected_files)
 
 if __name__ == '__main__':
     unittest.main()

combo_cases_and_deaths/tests/test_run.py

@@ -106,7 +106,8 @@ def run_module(params: Dict[str, Any]):
     for metric, geo_res, sensor, smoother in product(
         METRICS, GEO_RESOLUTIONS, SENSORS, SMOOTHERS
     ):
-        print(metric, geo_res, sensor, smoother)
+        if "cumulative" in sensor and "seven_day_average" in smoother:
+            continue
         logger.info(
             event="generating signal and exporting to CSV",
             metric=metric,

jhu/delphi_jhu/run.py

@@ -31,6 +31,8 @@ def test_output_files_exist(self, run_as_module):
         for date in dates:
             for geo in geos:
                 for metric in metrics:
+                    if "7dav" in metric and "cumulative" in metric:
+                        continue
                     # Can't compute 7dav for first few days of data because of NAs
                     if date > "20200305" or "7dav" not in metric:
                         expected_files += [date + "_" + geo + "_" + metric + ".csv"]

jhu/tests/test_run.py

@@ -9,14 +9,14 @@ def test_output_files_smoothed(self, run_as_module):
         dates = [str(x) for x in range(20200303, 20200310)]
 
         smoothed = pd.read_csv(
-            join("./receiving", f"{dates[-1]}_state_confirmed_7dav_cumulative_num.csv")
+            join("./receiving", f"{dates[-1]}_state_confirmed_7dav_incidence_num.csv")
         )
 
         # Build a dataframe out of the individual day files
         raw = pd.concat(
             [
                 pd.read_csv(
-                    join("./receiving", f"{date}_state_confirmed_cumulative_num.csv")
+                    join("./receiving", f"{date}_state_confirmed_incidence_num.csv")
                 )
                 for date in dates
             ]

jhu/tests/test_smooth.py

@@ -103,6 +103,8 @@ def run_module(params: Dict[str, Dict[str, Any]]):
     dfs = {metric: pull_usafacts_data(base_url, metric, logger) for metric in METRICS}
     for metric, geo_res, sensor, smoother in product(
             METRICS, GEO_RESOLUTIONS, SENSORS, SMOOTHERS):
+        if "cumulative" in sensor and "seven_day_average" in smoother:
+            continue
         logger.info("generating signal and exporting to CSV",
             geo_res = geo_res,
             metric = metric,

usafacts/delphi_usafacts/run.py

@@ -54,8 +54,9 @@ def test_output_files_exist(self):
                 for metric in metrics:
                     if "7dav" in metric and date in dates[:6]:
                         continue  # there are no 7dav signals for first 6 days
+                    if "7dav" in metric and "cumulative" in metric:
+                        continue
                     expected_files += [date + "_" + geo + "_" + metric + ".csv"]
-
         assert set(csv_files) == set(expected_files)
 
     def test_output_file_format(self):