Merge pull request #728 from benjaminysmith/even_more_logging

Add logging to quidel_covidtest and claims_hosp

Author: krivard

ansible/templates/quidel_covidtest-params-prod.json.j2

@@ -13,5 +13,5 @@
   "bucket_name": "{{ quidel_aws_bucket_name }}",
   "wip_signal": [""],
   "mode": "",
-  "log_filename": "/var/log/indicators/quidel.log"
+  "log_filename": "/var/log/indicators/quidel_covidtest.log"
 } 

claims_hosp/delphi_claims_hosp/run.py

@@ -6,12 +6,12 @@
 """
 
 # standard packages
-import logging
+import time
 from datetime import datetime, timedelta
 from pathlib import Path
 
 # third party
-from delphi_utils import read_params
+from delphi_utils import read_params, get_structured_logger
 
 # first party
 from .config import Config
@@ -20,8 +20,9 @@
 
 def run_module():
     """Read from params.json and generate updated claims-based hospitalization indicator values."""
+    start_time = time.time()
     params = read_params()
-    logging.basicConfig(level=logging.DEBUG)
+    logger = get_structured_logger(__name__, filename = params.get("log_filename"))
 
     # handle range of estimates to produce
     # filename expected to have format: EDI_AGG_INPATIENT_DDMMYYYY_HHMM{timezone}.csv.gz
@@ -46,32 +47,33 @@ def run_module():
         startdate = params['start_date']
 
     # print out information
-    logging.info("first sensor date:\t%s", startdate)
-    logging.info("last sensor date:\t%s", enddate)
-    logging.info("drop date:\t\t%s", dropdate)
-    logging.info("n_backfill_days:\t%s", params["n_backfill_days"])
-    logging.info("n_waiting_days:\t%s", params["n_waiting_days"])
-    logging.info("geos:\t\t\t%s", params["geos"])
-    logging.info("outpath:\t\t%s", params["export_dir"])
-    logging.info("parallel:\t\t%s", params["parallel"])
-    logging.info("weekday:\t\t%s", params["weekday"])
-    logging.info("write_se:\t\t%s", params["write_se"])
+    logger.info("Loaded params",
+                startdate = startdate,
+                enddate = enddate,
+                dropdate = dropdate,
+                n_backfill_days = params["n_backfill_days"],
+                n_waiting_days = params["n_waiting_days"],
+                geos = params["geos"],
+                outpath = params["export_dir"],
+                parallel = params["parallel"],
+                weekday = params["weekday"],
+                write_se = params["write_se"])
 
     # generate indicator csvs
     for geo in params["geos"]:
         for weekday in params["weekday"]:
             if weekday:
-                logging.info("starting %s, weekday adj", geo)
+                logger.info("starting weekday adj", geo = geo)
             else:
-                logging.info("starting %s, no adj", geo)
+                logger.info("starting no weekday adj", geo =  geo)
 
             signal_name = Config.signal_weekday_name if weekday else Config.signal_name
             if params["write_se"]:
                 assert params["obfuscated_prefix"] is not None, \
                     "supply obfuscated prefix in params.json"
                 signal_name = params["obfuscated_prefix"] + "_" + signal_name
 
-            logging.info("output signal name %s", signal_name)
+            logger.info("Updating signal name", signal_name = signal_name)
             updater = ClaimsHospIndicatorUpdater(
                 startdate,
                 enddate,
@@ -83,5 +85,8 @@ def run_module():
                 signal_name
             )
             updater.update_indicator(params["input_file"], params["export_dir"])
-        logging.info("finished %s", geo)
-    logging.info("finished all")
+        logger.info("finished updating", geo = geo)
+    
+    elapsed_time_in_seconds = round(time.time() - start_time, 2)
+    logger.info("Completed indicator run",
+        elapsed_time_in_seconds = elapsed_time_in_seconds)

quidel_covidtest/delphi_quidel_covidtest/run.py

@@ -4,7 +4,12 @@
 This module should contain a function called `run_module`, that is executed
 when the module is run with `python -m MODULE_NAME`.
 """
-from delphi_utils import read_params, add_prefix, create_export_csv
+from delphi_utils import (
+    read_params,
+    add_prefix,
+    create_export_csv,
+    get_structured_logger
+)
 
 from .geo_maps import geo_map
 from .pull import (pull_quidel_covidtest,
@@ -18,10 +23,13 @@
                         SMOOTHED_TEST_PER_DEVICE, RAW_TEST_PER_DEVICE,
                         GEO_RESOLUTIONS, SENSORS, SMOOTHERS)
 
+import time
 
 def run_module():
     """Run the quidel_covidtest indicator."""
+    start_time = time.time()
     params = read_params()
+    logger = get_structured_logger(__name__, filename = params.get("log_filename"))
     cache_dir = params["cache_dir"]
     export_dir = params["export_dir"]
     export_start_date = params["export_start_date"]
@@ -84,3 +92,7 @@ def run_module():
     # Export the cache file if the pipeline runs successfully.
     # Otherwise, don't update the cache file
     update_cache_file(df, _end_date, cache_dir)
+
+    elapsed_time_in_seconds = round(time.time() - start_time, 2)
+    logger.info("Completed indicator run",
+        elapsed_time_in_seconds = elapsed_time_in_seconds)