Merge pull request #803 from cmu-delphi/claims-hosp-params

Update claims hosp params

Author: krivard

claims_hosp/delphi_claims_hosp/run.py

@@ -23,72 +23,73 @@ def run_module():
     start_time = time.time()
     params = read_params()
     logger = get_structured_logger(
-        __name__, filename=params.get("log_filename"),
-        log_exceptions=params.get("log_exceptions", True))
+        __name__, filename=params["common"].get("log_filename"),
+        log_exceptions=params["common"].get("log_exceptions", True))
 
     # handle range of estimates to produce
     # filename expected to have format: EDI_AGG_INPATIENT_DDMMYYYY_HHMM{timezone}.csv.gz
-    if params["drop_date"] is None:
+    if params["indicator"]["drop_date"] is None:
         dropdate_dt = datetime.strptime(
-            Path(params["input_file"]).name.split("_")[3], "%d%m%Y")
+            Path(params["indicator"]["input_file"]).name.split("_")[3], "%d%m%Y")
     else:
-        dropdate_dt = datetime.strptime(params["drop_date"], "%Y-%m-%d")
+        dropdate_dt = datetime.strptime(params["indicator"]["drop_date"], "%Y-%m-%d")
 
     # produce estimates for n_backfill_days
     # most recent n_waiting_days won't be est
-    enddate_dt = dropdate_dt - timedelta(days=params["n_waiting_days"])
-    startdate_dt = enddate_dt - timedelta(days=params["n_backfill_days"])
+    enddate_dt = dropdate_dt - timedelta(days=params["indicator"]["n_waiting_days"])
+    startdate_dt = enddate_dt - timedelta(days=params["indicator"]["n_backfill_days"])
     enddate = str(enddate_dt.date())
     startdate = str(startdate_dt.date())
     dropdate = str(dropdate_dt.date())
 
     # now allow manual overrides
-    if params["end_date"] is not None:
-        enddate = params["end_date"]
-    if params["start_date"] is not None:
-        startdate = params['start_date']
+    if params["indicator"]["end_date"] is not None:
+        enddate = params["indicator"]["end_date"]
+    if params["indicator"]["start_date"] is not None:
+        startdate = params["indicator"]['start_date']
 
     # print out information
     logger.info("Loaded params",
                 startdate = startdate,
                 enddate = enddate,
                 dropdate = dropdate,
-                n_backfill_days = params["n_backfill_days"],
-                n_waiting_days = params["n_waiting_days"],
-                geos = params["geos"],
-                outpath = params["export_dir"],
-                parallel = params["parallel"],
-                weekday = params["weekday"],
-                write_se = params["write_se"])
+                n_backfill_days = params["indicator"]["n_backfill_days"],
+                n_waiting_days = params["indicator"]["n_waiting_days"],
+                geos = params["indicator"]["geos"],
+                outpath = params["common"]["export_dir"],
+                parallel = params["indicator"]["parallel"],
+                weekday = params["indicator"]["weekday"],
+                write_se = params["indicator"]["write_se"])
 
     # generate indicator csvs
-    for geo in params["geos"]:
-        for weekday in params["weekday"]:
+    for geo in params["indicator"]["geos"]:
+        for weekday in params["indicator"]["weekday"]:
             if weekday:
                 logger.info("starting weekday adj", geo = geo)
             else:
                 logger.info("starting no weekday adj", geo =  geo)
 
             signal_name = Config.signal_weekday_name if weekday else Config.signal_name
-            if params["write_se"]:
-                assert params["obfuscated_prefix"] is not None, \
+            if params["indicator"]["write_se"]:
+                assert params["indicator"]["obfuscated_prefix"] is not None, \
                     "supply obfuscated prefix in params.json"
-                signal_name = params["obfuscated_prefix"] + "_" + signal_name
+                signal_name = params["indicator"]["obfuscated_prefix"] + "_" + signal_name
 
             logger.info("Updating signal name", signal_name = signal_name)
             updater = ClaimsHospIndicatorUpdater(
                 startdate,
                 enddate,
                 dropdate,
                 geo,
-                params["parallel"],
+                params["indicator"]["parallel"],
                 weekday,
-                params["write_se"],
+                params["indicator"]["write_se"],
                 signal_name
             )
-            updater.update_indicator(params["input_file"], params["export_dir"])
+            updater.update_indicator(params["indicator"]["input_file"],
+                                     params["common"]["export_dir"])
         logger.info("finished updating", geo = geo)
-    
+
     elapsed_time_in_seconds = round(time.time() - start_time, 2)
     logger.info("Completed indicator run",
         elapsed_time_in_seconds = elapsed_time_in_seconds)

claims_hosp/params.json.template

@@ -1,17 +1,19 @@
 {
-  "static_file_dir": "./static",
-  "export_dir": "./receiving",
-  "cache_dir": "./cache",
-  "input_file": "./tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz",
-  "start_date": "2020-02-01",
-  "end_date": null,
-  "drop_date": null,
-  "n_backfill_days": 60,
-  "n_waiting_days": 3,
-  "write_se": false,
-  "obfuscated_prefix": "foo_obfuscated",
-  "parallel": false,
-  "geos": ["state", "msa", "hrr", "county"],
-  "weekday": [true, false],
-  "log_exceptions": false
+  "common": {
+    "export_dir": "./receiving",
+    "log_exceptions": false
+  },
+  "indicator": {
+    "input_file": "./tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz",
+    "start_date": "2020-02-01",
+    "end_date": null,
+    "drop_date": null,
+    "n_backfill_days": 60,
+    "n_waiting_days": 3,
+    "write_se": false,
+    "obfuscated_prefix": "foo_obfuscated",
+    "parallel": false,
+    "geos": ["state", "msa", "hrr", "county"],
+    "weekday": [true, false]
+  }
 }

claims_hosp/tests/params.json.template

@@ -1,11 +1,13 @@
 {
-  "static_file_dir": "../static",
-  "export_dir": "./receiving",
-  "cache_dir": "./cache",
-  "input_file": "test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz",
-  "drop_date": "2020-06-11",
-  "n_backfill_days": 60,
-  "n_waiting_days": 3,
-  "write_se": false,
-  "obfuscated_prefix": "foo_obfuscated"
+  "common": {
+    "export_dir": "./receiving"
+  },
+  "indicator": {
+    "input_file": "test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz",
+    "drop_date": "2020-06-11",
+    "n_backfill_days": 60,
+    "n_waiting_days": 3,
+    "write_se": false,
+    "obfuscated_prefix": "foo_obfuscated"
+  }
 }

claims_hosp/tests/test_indicator.py

@@ -12,8 +12,8 @@
 
 CONFIG = Config()
 PARAMS = read_params()
-DATA_FILEPATH = PARAMS["input_file"]
-DROP_DATE = pd.to_datetime(PARAMS["drop_date"])
+DATA_FILEPATH = PARAMS["indicator"]["input_file"]
+DROP_DATE = pd.to_datetime(PARAMS["indicator"]["drop_date"])
 
 
 class TestLoadData:

claims_hosp/tests/test_load_data.py

@@ -10,8 +10,8 @@
 CONFIG = Config()
 CONSTANTS = GeoConstants()
 PARAMS = read_params()
-DATA_FILEPATH = PARAMS["input_file"]
-DROP_DATE = pd.to_datetime(PARAMS["drop_date"])
+DATA_FILEPATH = PARAMS["indicator"]["input_file"]
+DROP_DATE = pd.to_datetime(PARAMS["indicator"]["drop_date"])
 
 
 class TestLoadData:

claims_hosp/tests/test_update_indicator.py

@@ -17,8 +17,8 @@
 CONFIG = Config()
 CONSTANTS = GeoConstants()
 PARAMS = read_params()
-DATA_FILEPATH = PARAMS["input_file"]
-DROP_DATE = pd.to_datetime(PARAMS["drop_date"])
+DATA_FILEPATH = PARAMS["indicator"]["input_file"]
+DROP_DATE = pd.to_datetime(PARAMS["indicator"]["drop_date"])
 OUTPATH = "test_data/"
 
 
@@ -174,7 +174,7 @@ def test_write_to_csv_results(self):
         td.cleanup()
 
     def test_write_to_csv_with_se_results(self):
-        obfuscated_name = PARAMS["obfuscated_prefix"]
+        obfuscated_name = PARAMS["indicator"]["obfuscated_prefix"]
         signal_name = obfuscated_name + "_" + Config.signal_weekday_name
         updater = ClaimsHospIndicatorUpdater(
             "02-01-2020",