Merge pull request #1199 from cmu-delphi/pop_prop_signal

HHS data:  population-proportionate variants

Author: krivard

hhs_hosp/delphi_hhs/constants.py

@@ -3,11 +3,14 @@
 
 CONFIRMED = "confirmed_admissions_covid_1d"
 SUM_CONF_SUSP = "sum_confirmed_suspected_admissions_covid_1d"
-
+CONFIRMED_PROP = "confirmed_admissions_covid_1d_prop"
+SUM_CONF_SUSP_PROP = "sum_confirmed_suspected_admissions_covid_1d_prop"
 
 SIGNALS = [
     CONFIRMED,
-    SUM_CONF_SUSP
+    SUM_CONF_SUSP,
+    CONFIRMED_PROP,
+    SUM_CONF_SUSP_PROP
 ]
 
 GEOS = [

hhs_hosp/delphi_hhs/run.py

@@ -17,7 +17,6 @@
 
 from .constants import SIGNALS, GEOS, SMOOTHERS, CONFIRMED, SUM_CONF_SUSP
 
-
 def _date_to_int(d):
     """Return a date object as a yyyymmdd int."""
     return int(d.strftime("%Y%m%d"))
@@ -98,12 +97,13 @@ def run_module(params):
     all_columns = pd.concat(dfs)
     geo_mapper = GeoMapper()
     stats = []
-
     for sensor, smoother, geo in product(SIGNALS, SMOOTHERS, GEOS):
         df = geo_mapper.add_geocode(make_signal(all_columns, sensor),
                                     "state_id",
                                     "state_code",
                                     from_col="state")
+        if sensor.endswith("_prop"):
+            df=pop_proportion(df, geo_mapper)
         df = make_geo(df, geo, geo_mapper)
         df = smooth_values(df, smoother[0])
         if df.empty:
@@ -139,6 +139,12 @@ def smooth_values(df, smoother):
     )
     return df
 
+def pop_proportion(df,geo_mapper):
+    """Get the population-proportionate variants as the dataframe val."""
+    pop_val=geo_mapper.add_population_column(df, "state_code")
+    df["val"]=round(df["val"]/pop_val["population"]*100000, 7)
+    pop_val.drop("population", axis=1, inplace=True)
+    return df
 
 def make_geo(state, geo, geo_mapper):
     """Transform incoming geo (state) to another geo."""
@@ -158,15 +164,15 @@ def make_signal(all_columns, sig):
     """Generate column sums according to signal name."""
     assert sig in SIGNALS, f"Unexpected signal name '{sig}';" + \
         " familiar names are '{', '.join(SIGNALS)}'"
-    if sig == CONFIRMED:
+    if sig.startswith(CONFIRMED):
         df = pd.DataFrame({
             "state": all_columns.state.apply(str.lower),
             "timestamp":int_date_to_previous_day_datetime(all_columns.date),
             "val": \
             all_columns.previous_day_admission_adult_covid_confirmed + \
             all_columns.previous_day_admission_pediatric_covid_confirmed
         })
-    elif sig == SUM_CONF_SUSP:
+    elif sig.startswith(SUM_CONF_SUSP):
         df = pd.DataFrame({
             "state": all_columns.state.apply(str.lower),
             "timestamp":int_date_to_previous_day_datetime(all_columns.date),

hhs_hosp/tests/test_run.py

@@ -5,8 +5,8 @@
 import os
 
 from delphi_hhs.run import _date_to_int, int_date_to_previous_day_datetime, generate_date_ranges, \
-    make_signal, make_geo, run_module
-from delphi_hhs.constants import CONFIRMED, SUM_CONF_SUSP, SMOOTHERS, GEOS, SIGNALS
+    make_signal, make_geo, run_module, pop_proportion
+from delphi_hhs.constants import CONFIRMED, SUM_CONF_SUSP, SMOOTHERS, GEOS, SIGNALS, CONFIRMED_PROP, SUM_CONF_SUSP_PROP
 from delphi_utils.geomap import GeoMapper
 from freezegun import freeze_time
 import numpy as np
@@ -66,17 +66,49 @@ def test_make_signal():
         'val': [5.],
     })
     pd.testing.assert_frame_equal(expected_confirmed, make_signal(data, CONFIRMED))
+    pd.testing.assert_frame_equal(expected_confirmed, make_signal(data, CONFIRMED_PROP))
 
     expected_sum = pd.DataFrame({
         'state': ['na'],
         'timestamp': [datetime(year=2020, month=1, day=1)],
         'val': [15.],
     })
     pd.testing.assert_frame_equal(expected_sum, make_signal(data, SUM_CONF_SUSP))
+    pd.testing.assert_frame_equal(expected_sum, make_signal(data, SUM_CONF_SUSP_PROP))
 
     with pytest.raises(Exception):
         make_signal(data, "zig")
 
+def test_pop_proportion():
+    geo_mapper = GeoMapper()
+    test_df = pd.DataFrame({  
+        'state': ['PA'],
+        'state_code': [42],
+        'timestamp': [datetime(year=2020, month=1, day=1)],
+        'val': [15.],})
+    pd.testing.assert_frame_equal(
+        pop_proportion(test_df, geo_mapper),
+        pd.DataFrame({
+            'state': ['PA'],
+            'state_code': [42],
+            'timestamp': [datetime(year=2020, month=1, day=1)],
+            'val': [0.1171693],})
+    )
+
+    test_df= pd.DataFrame({  
+        'state': ['WV'],
+        'state_code': [54],
+        'timestamp': [datetime(year=2020, month=1, day=1)],
+        'val': [150.],})
+
+    pd.testing.assert_frame_equal(
+        pop_proportion(test_df, geo_mapper),
+        pd.DataFrame({
+            'state': ['WV'],
+            'state_code': [54],
+            'timestamp': [datetime(year=2020, month=1, day=1)],
+            'val': [8.3698491],})
+    )
 
 def test_make_geo():
     """Check that geographies transform correctly."""