Merge pull request #612 from benjaminysmith/SirCAL

Add structured logging to SirCAL

Author: krivard

_delphi_utils_python/delphi_utils/__init__.py

@@ -7,6 +7,7 @@
 from .export import create_export_csv
 from .utils import read_params
 
+from .logger import get_structured_logger
 from .geomap import GeoMapper
 from .smooth import Smoother
 from .signal import add_prefix, public_signal

sir_complainsalot/delphi_sir_complainsalot/check_source.py

@@ -1,10 +1,13 @@
 from dataclasses import dataclass
 from typing import List
+from delphi_utils import get_structured_logger
 
 import covidcast
 import numpy as np
 import pandas as pd
 
+logger = get_structured_logger(__name__)
+
 @dataclass
 class Complaint:
     message: str
@@ -83,6 +86,14 @@ def check_source(data_source, meta, params, grace):
             # No gap detection for this source
             continue
 
+        logger.info("Retrieving signal",
+                    source=data_source,
+                    signal=row["signal"],
+                    start_day=(row["max_time"] -
+                               gap_window).strftime("%Y-%m-%d"),
+                    end_day=row["max_time"].strftime("%Y-%m-%d"),
+                    geo_type=row["geo_type"])
+
         latest_data = covidcast.signal(
             data_source, row["signal"],
             start_day=row["max_time"] - gap_window,

sir_complainsalot/delphi_sir_complainsalot/run.py

@@ -5,6 +5,8 @@
 when the module is run with `python -m delphi_sir_complainsalot`.
 """
 
+import logging
+import structlog
 import sys
 
 from itertools import groupby
@@ -13,10 +15,12 @@
 from slack.errors import SlackApiError
 
 from delphi_utils import read_params
+from delphi_utils import get_structured_logger
 import covidcast
 
 from .check_source import check_source
 
+logger = get_structured_logger(__name__)
 
 def run_module():
 
@@ -30,7 +34,12 @@ def run_module():
 
     if len(complaints) > 0:
         for complaint in complaints:
-            print(complaint)
+            logger.critical(event="signal out of SLA",
+                            message=complaint.message,
+                            data_source=complaint.data_source,
+                            signal=complaint.signal,
+                            geo_types=complaint.geo_types,
+                            last_updated=complaint.last_updated.strftime("%Y-%m-%d"))
 
         report_complaints(complaints, params)
 
@@ -46,14 +55,14 @@ def split_complaints(complaints, n=49):
 def report_complaints(all_complaints, params):
     """Post complaints to Slack."""
     if not params["slack_token"]:
-        print("\b (dry-run)")
+        logger.info("(dry-run)")
         return
 
     client = WebClient(token=params["slack_token"])
 
     for complaints in split_complaints(all_complaints):
         blocks = format_complaints_aggregated_by_source(complaints)
-        print(f"blocks: {len(blocks)}")
+        logger.info(f"blocks: {len(blocks)}")
         try:
             client.chat_postMessage(
                 channel=params["channel"],