Aggregate SirCAL messages by source and error

sir_complainsalot/delphi_sir_complainsalot/run.py

@@ -7,6 +7,8 @@
 
 import sys
 
+from itertools import groupby
+
 from slack import WebClient
 from slack.errors import SlackApiError
 
@@ -15,14 +17,16 @@
 
 from .check_source import check_source
 
+
 def run_module():
 
     params = read_params()
     meta = covidcast.metadata()
 
     complaints = []
     for data_source in params["sources"].keys():
-        complaints.extend(check_source(data_source, meta, params["sources"], params.get("grace", 0)))
+        complaints.extend(check_source(data_source, meta,
+                                       params["sources"], params.get("grace", 0)))
 
     if len(complaints) > 0:
         for complaint in complaints:
@@ -32,11 +36,13 @@ def run_module():
 
         sys.exit(1)
 
+
 def split_complaints(complaints, n=49):
     """Yield successive n-sized chunks from complaints list."""
     for i in range(0, len(complaints), n):
         yield complaints[i:i + n]
 
+
 def report_complaints(all_complaints, params):
     """Post complaints to Slack."""
     if not params["slack_token"]:
@@ -46,7 +52,7 @@ def report_complaints(all_complaints, params):
     client = WebClient(token=params["slack_token"])
 
     for complaints in split_complaints(all_complaints):
-        blocks = format_complaints(complaints)
+        blocks = format_complaints_aggregated_by_source(complaints)
         print(f"blocks: {len(blocks)}")
         try:
             client.chat_postMessage(
@@ -57,6 +63,66 @@ def report_complaints(all_complaints, params):
             # You will get a SlackApiError if "ok" is False
             assert False, e.response["error"]
 
+def get_maintainers_block(complaints):
+    """Build a Slack block to alert maintainers to pay attention."""
+    maintainers = set()
+    for c in complaints:
+        maintainers.update(c.maintainers)
+
+    maintainers_block = {
+        "type": "section",
+        "text": {
+                "type": "mrkdwn",
+                "text": "Hi, this is Sir Complains-a-Lot. I need to speak to " +
+                        (", ".join("<@{0}>".format(m)
+                                   for m in maintainers)) + "."
+        }
+    }
+
+    return maintainers_block
+
+
+def format_complaints_aggregated_by_source(complaints):
+    """Build formatted Slack message for posting to the API, aggregating
+    complaints by source to reduce the number of blocks."""
+
+    blocks = [get_maintainers_block(complaints)]
+
+    def message_for_source(complaint):
+        return "{main_text} - (last update: {last_updated})".format(
+            main_text=complaint.message,
+            last_updated=complaint.last_updated.strftime("%Y-%m-%d"))
+
+    for source, complaints_by_source in groupby(
+            complaints, key=lambda x: x.data_source):
+        for message, complaint_list in groupby(
+                complaints_by_source, key=message_for_source):
+            signal_and_geo_types = ""
+            for complaint in complaint_list:
+                signal_and_geo_types += "`{signal}: [{geo_types}]`\n".format(
+                    signal=complaint.signal,
+                    geo_types=", ".join(complaint.geo_types))
+
+            blocks.extend([
+                {
+                    "type": "divider"
+                },
+                {
+                    "type": "section",
+                    "text": {
+                        "type": "mrkdwn",
+                        "text": "*{source_name}* {message}:\n{signals}"
+                        .format(
+                            source_name=source.upper(),
+                            message=message,
+                            signals=signal_and_geo_types)
+                    }
+                }
+            ])
+
+    return blocks
+
+
 def format_complaints(complaints):
     """Build a formatted Slack message for posting to the API.
 
@@ -65,31 +131,17 @@ def format_complaints(complaints):
 
     """
 
-    maintainers = set()
-    for c in complaints:
-        maintainers.update(c.maintainers)
-
-    blocks = [
-	{
-	    "type": "section",
-	    "text": {
-		"type": "mrkdwn",
-		"text": "Hi, this is Sir Complains-a-Lot. I need to speak to " +
-                        (", ".join("<@{0}>".format(m) for m in maintainers)) + "."
-	    }
-	}
-    ]
+    blocks = [get_maintainers_block(complaints)]
 
     for complaint in complaints:
         blocks.append(
             {
-	        "type": "section",
-		"text": {
-		    "type": "mrkdwn",
-		    "text": complaint.to_md()
-		}
-	    }
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": complaint.to_md()
+                }
+            }
         )
 
-
     return blocks