Merge pull request #223 from cmu-delphi/handle_wip_signal_ccad

Combo cases & deaths: standardizing signal names

Author: krivard

combo_cases_and_deaths/delphi_combo_cases_and_deaths/constants.py

@@ -0,0 +1,21 @@
+"""Registry for signal names, geo types and other constants"""
+METRICS = [
+    "confirmed",
+    "deaths",
+]
+SMOOTH_TYPES = [
+    "",
+    "7dav",
+]
+SENSORS = [
+    "incidence_num",
+    "cumulative_num",
+    "incidence_prop",
+    "cumulative_prop",
+]
+GEO_RESOLUTIONS = [
+    "county",
+    "state",
+    "msa",
+    "hrr",
+]

combo_cases_and_deaths/delphi_combo_cases_and_deaths/handle_wip_signal.py

@@ -0,0 +1,62 @@
+"""
+Handle signal names.
+
+Author: Vishakha
+Created: 2020-08-07
+"""
+
+import covidcast
+
+
+def add_prefix(signal_names, wip_signal, prefix="wip_"):
+    """Adds prefix to signal if there is a WIP signal
+    Parameters
+    ----------
+    signal_names: List[str]
+        Names of signals to be exported
+    prefix : 'wip_'
+        prefix for new/non public signals
+    wip_signal : List[str] or bool
+        a list of wip signals: [], OR
+        all signals in the registry: True OR
+        only signals that have never been published: False
+    Returns
+    -------
+    List of signal names
+        wip/non wip signals for further computation
+    """
+
+    if wip_signal is True:
+        return [prefix + signal for signal in signal_names]
+    if isinstance(wip_signal, list):
+        make_wip = set(wip_signal)
+        return[
+            prefix + signal if signal in make_wip else signal
+            for signal in signal_names
+        ]
+    if wip_signal in {False, ""}:
+        return [
+            signal if public_signal(signal)
+            else prefix + signal
+            for signal in signal_names
+        ]
+    raise ValueError("Supply True | False or '' or [] | list()")
+
+
+def public_signal(signal_):
+    """Checks if the signal name is already public using COVIDcast
+    Parameters
+    ----------
+    signal_ : str
+        Name of the signal
+    Returns
+    -------
+    bool
+        True if the signal is present
+        False if the signal is not present
+    """
+    epidata_df = covidcast.metadata()
+    for index in range(len(epidata_df)):
+        if epidata_df['signal'][index] == signal_:
+            return True
+    return False

combo_cases_and_deaths/delphi_combo_cases_and_deaths/run.py

@@ -1,13 +1,10 @@
 # -*- coding: utf-8 -*-
 """Functions to call when running the function.
-
 This module should contain a function called `run_module`, that is executed when
 the module is run with `python -m delphi_combo_cases_and_deaths`.
-
 This module produces a combined signal for jhu-csse and usa-facts.  This signal
 is only used for visualization.  It sources Puerto Rico from jhu-csse and
 everything else from usa-facts.
-
 """
 from datetime import date, timedelta, datetime
 from itertools import product
@@ -18,29 +15,10 @@
 import pandas as pd
 
 from delphi_utils import read_params, create_export_csv
+from .constants import *
+from .handle_wip_signal import *
 
 
-METRICS = [
-    "confirmed",
-    "deaths",
-]
-SMOOTH_TYPES = [
-    "",
-    "7dav",
-]
-SENSORS = [
-    "incidence_num",
-    "cumulative_num",
-    "incidence_prop",
-    "cumulative_prop",
-]
-GEO_RESOLUTIONS = [
-    "county",
-    "state",
-    "msa",
-    "hrr",
-]
-
 def check_not_none(data_frame, label, date_range):
     """Exit gracefully if a data frame we attempted to retrieve is empty"""
     if data_frame is None:
@@ -78,7 +56,6 @@ def extend_raw_date_range(params, sensor_name):
     """A complete issue includes smoothed signals as well as all raw data
     that contributed to the smoothed values, so that it's possible to use
     the raw values in the API to reconstruct the smoothed signal at will.
-
     The smoother we're currently using incorporates the previous 7
     days of data, so we must extend the date range of the raw data
     backwards by 7 days.
@@ -155,11 +132,21 @@ def run_module():
         params['date_range'] = [date1, date2]
 
     for metric, geo_res, sensor_name, signal in variants:
-        create_export_csv(
-            combine_usafacts_and_jhu(signal, geo_res, extend_raw_date_range(params, sensor_name)),
-            export_dir=params['export_dir'],
-            start_date=pd.to_datetime(params['export_start_date']),
-            metric=metric,
-            geo_res=geo_res,
-            sensor=sensor_name,
-        )
+
+        df = combine_usafacts_and_jhu(signal, geo_res, extend_raw_date_range(params, sensor_name))
+
+        df = df.copy()
+        df["timestamp"] = pd.to_datetime(df["timestamp"])
+        start_date = pd.to_datetime(params['export_start_date'])
+        export_dir = params["export_dir"]
+        dates = pd.Series(
+            df[df["timestamp"] >= start_date]["timestamp"].unique()
+        ).sort_values()
+
+        signal_name = add_prefix([signal], wip_signal=params["wip_signal"], prefix="wip_")
+        for date_ in dates:
+            export_fn = f'{date_.strftime("%Y%m%d")}_{geo_res}_' f"{signal_name[0]}.csv"
+            df[df["timestamp"] == date_][["geo_id", "val", "se", "sample_size", ]].to_csv(
+                f"{export_dir}/{export_fn}", index=False, na_rep="NA"
+            )
+

combo_cases_and_deaths/params.json.template

@@ -4,5 +4,6 @@
   "cache_dir": "./cache",
   "export_start_date":[2020,4,1],
   "date_range":"new",
-  "source":"indicator-combination"
+  "source":"indicator-combination",
+  "wip_signal": ""
 }

combo_cases_and_deaths/tests/params.json.template

@@ -1,5 +1,9 @@
 {
-  "static_file_dir": "../static",
+  "static_file_dir": "./static",
   "export_dir": "./receiving",
-  "cache_dir": "./cache"
+  "cache_dir": "./cache",
+  "export_start_date":[2020,4,1],
+  "date_range": "",
+  "source":"",
+  "wip_signal": ""
 }

combo_cases_and_deaths/tests/test_run.py

@@ -1,18 +1,23 @@
 from datetime import date
 from itertools import product
 import pytest
+import unittest
+
+from delphi_combo_cases_and_deaths.run import extend_raw_date_range, sensor_signal
+from delphi_combo_cases_and_deaths.handle_wip_signal import add_prefix
+from delphi_utils import read_params
+from delphi_combo_cases_and_deaths.constants import *
 
-from delphi_combo_cases_and_deaths.run import extend_raw_date_range,sensor_signal,METRICS,SENSORS,SMOOTH_TYPES
 
 def test_issue_dates():
-    reference_dr = [date.today(),date.today()]
+    reference_dr = [date.today(), date.today()]
     params = {'date_range': reference_dr}
     n_changed = 0
     variants = [sensor_signal(metric, sensor, smoother) for
                 metric, sensor, smoother in
-                product(METRICS,SENSORS,SMOOTH_TYPES)]
+                product(METRICS, SENSORS, SMOOTH_TYPES)]
     variants_changed = []
-    for sensor_name,signal in variants:
+    for sensor_name, signal in variants:
         dr = extend_raw_date_range(params, sensor_name)
         if dr[0] != reference_dr[0]:
             n_changed += 1
@@ -21,3 +26,33 @@ def test_issue_dates():
 All variants: {variants}
 Date-extended variants: {variants_changed}
 """
+
+
+def test_handle_wip_signal():
+
+    signal_list = [sensor_signal(metric, sensor, smoother)[1]
+                   for (metric, sensor, smoother) in
+                   product(METRICS, SENSORS, SMOOTH_TYPES)]
+
+    # Test wip_signal = True (all signals should receive prefix)
+    signal_names = add_prefix(signal_list, True, prefix="wip_")
+    assert all(s.startswith("wip_") for s in signal_names)
+
+    # Test wip_signal = list (only listed signals should receive prefix)
+    signal_names = add_prefix(signal_list, [signal_list[0]], prefix="wip_")
+    print(signal_names)
+    assert signal_names[0].startswith("wip_")
+    assert all(not s.startswith("wip_") for s in signal_names[1:])
+
+    # Test wip_signal = False (only unpublished signals should receive prefix)
+    signal_names = add_prefix(["xyzzy", signal_list[0]], False, prefix="wip_")
+    assert signal_names[0].startswith("wip_")
+    assert all(not s.startswith("wip_") for s in signal_names[1:])
+
+
+class MyTestCase(unittest.TestCase):
+    pass
+
+
+if __name__ == '__main__':
+    unittest.main()