Merge pull request #929 from cmu-delphi/main

Deploy correct hush for SirCAL to production

Author: krivard

_delphi_utils_python/delphi_utils/validator/README.md

@@ -53,8 +53,8 @@ Please update the follow settings:
 
 * `common`: global validation settings
    * `data_source`: should match the [formatting](https://cmu-delphi.github.io/delphi-epidata/api/covidcast_signals.html) as used in COVIDcast API calls
-   * `end_date`: specifies the last date to be checked; if set to "latest", `end_date` will always be the current date
-   * `span_length`: specifies the number of days before the `end_date` to check. `span_length` should be long enough to contain all recent source data that is still in the process of being updated (i.e. in the backfill period), for example, if the data source of interest has a 2-week lag before all reports are in for a given date, `scan_length` should be 14 days
+   * `end_date`: specifies the last date to be checked; this can be specified as `YYYY-MM-DD` or as `today-{num}`.  The latter is interpretted as `num` days before the current date (with `today-0` being today).
+   * `span_length`: specifies the number of days before the `end_date` to check. `span_length` should be long enough to contain all recent source data that is still in the process of being updated (i.e. in the backfill period), for example, if the data source of interest has a 2-week lag before all reports are in for a given date, `span_length` should be 14 days
    * `suppressed_errors`: list of objects specifying errors that have been manually verified as false positives or acceptable deviations from expected.  These errors can be specified with the following variables, where omitted values are interpreted as a wildcard, i.e., not specifying a date applies to all dates:
        * `check_name` (required):  name of the check, as specified in the validation output
        * `date`:  date in `YYYY-MM-DD` format

_delphi_utils_python/delphi_utils/validator/utils.py

@@ -35,8 +35,9 @@ def __post_init__(self):
     def from_params(cls, end_date_str: str, span_length_int: int):
         """Create a TimeWindow from param representations of its members."""
         span_length = timedelta(days=span_length_int)
-        if end_date_str == "latest":
-            end_date = date.today()
+        if end_date_str.startswith("today-"):
+            days_back = timedelta(days=int(end_date_str[6:]))
+            end_date = date.today() - days_back
         else:
             end_date = datetime.strptime(end_date_str, '%Y-%m-%d').date()
         return cls(end_date, span_length)

_delphi_utils_python/tests/validator/test_utils.py

@@ -45,5 +45,8 @@ def test_string_init(self):
         """Test that TimeWindows can be derived from strings."""
         window = TimeWindow.from_params("2020-08-23", 366)
         assert window.start_date == date(2019, 8, 23)
-        latest_window = TimeWindow.from_params("latest", 1897)
+        latest_window = TimeWindow.from_params("today-0", 1897)
         assert latest_window.start_date == date(2014, 12, 5)
+        latest_window = TimeWindow.from_params("today-10", 3)
+        assert latest_window.end_date == date(2020, 2, 4)
+        assert latest_window.start_date == date(2020, 2, 1)

ansible/templates/sir_complainsalot-params-prod.json.j2

@@ -1,5 +1,4 @@
 {
-  "channel": "#sir-complains-a-lot",
   "log_filename": "/var/log/indicators/sircal.log",
   "slack_token": "{{ sir_complainsalot_slack_token }}",
   "sources": {

facebook/delphiFacebook/R/binary.R

@@ -115,7 +115,9 @@ get_binary_indicators <- function() {
     "smoothed_wvaccine_likely_govt_health", "weight", "v_vaccine_likely_govt_health", 6, compute_binary_response, jeffreys_binary,
     "smoothed_vaccine_likely_politicians", "weight_unif", "v_vaccine_likely_politicians", 6, compute_binary_response, jeffreys_binary,
     "smoothed_wvaccine_likely_politicians", "weight", "v_vaccine_likely_politicians", 6, compute_binary_response, jeffreys_binary,
-
+    "smoothed_vaccine_likely_doctors", "weight_unif", "v_vaccine_likely_doctors", 6, compute_binary_response, jeffreys_binary,
+    "smoothed_wvaccine_likely_doctors", "weight", "v_vaccine_likely_doctors", 6, compute_binary_response, jeffreys_binary,
+    
     # vaccine hesitancy reasons
     "smoothed_hesitancy_reason_sideeffects", "weight_unif", "v_hesitancy_reason_sideeffects", 6, compute_binary_response, jeffreys_binary,
     "smoothed_whesitancy_reason_sideeffects", "weight", "v_hesitancy_reason_sideeffects", 6, compute_binary_response, jeffreys_binary,

facebook/delphiFacebook/R/variables.R

@@ -342,16 +342,25 @@ code_vaccines <- function(input_data) {
 
   if ("V4_1" %in% names(input_data)) {
     input_data$v_vaccine_likely_friends <- input_data$V4_1 == 1
-    input_data$v_vaccine_likely_local_health <- input_data$V4_2 == 1
     input_data$v_vaccine_likely_who <- input_data$V4_3 == 1
     input_data$v_vaccine_likely_govt_health <- input_data$V4_4 == 1
     input_data$v_vaccine_likely_politicians <- input_data$V4_5 == 1
+    
+    if (wave < 8) {
+      input_data$v_vaccine_likely_local_health <- input_data$V4_2 == 1
+      input_data$v_vaccine_likely_doctors <- NA_real_
+    } else {
+      input_data$v_vaccine_likely_local_health <- NA_real_
+      input_data$v_vaccine_likely_doctors <- input_data$V4_2 == 1
+    }
+    
   } else {
     input_data$v_vaccine_likely_friends <- NA_real_
     input_data$v_vaccine_likely_local_health <- NA_real_
     input_data$v_vaccine_likely_who <- NA_real_
     input_data$v_vaccine_likely_govt_health <- NA_real_
     input_data$v_vaccine_likely_politicians <- NA_real_
+    input_data$v_vaccine_likely_doctors <- NA_real_
   }
 
   if ("V5a" %in% names(input_data) && "V5b" %in% names(input_data) && "V5c" %in% names(input_data)) {

sir_complainsalot/delphi_sir_complainsalot/run.py

@@ -50,15 +50,13 @@ def split_complaints(complaints, n=49):
 
 
 def report_complaints(all_complaints, slack_notifier):
-    """Post complaints to Slack."""
-    if not slack_notifier:
-        print("(dry-run)")
-        return
+    """Log complaints and optionally post to Slack."""
 
     for complaints in split_complaints(all_complaints):
-        blocks = format_complaints_aggregated_by_source(complaints)
-        print(f"blocks: {len(blocks)}")
-        slack_notifier.post_message(blocks)
+        blocks = format_and_log_complaints_aggregated_by_source(complaints)
+
+        if slack_notifier:
+            slack_notifier.post_message(blocks)
 
 def get_maintainers_block(complaints):
     """Build a Slack block to alert maintainers to pay attention."""
@@ -79,7 +77,7 @@ def get_maintainers_block(complaints):
     return maintainers_block
 
 
-def format_complaints_aggregated_by_source(complaints):
+def format_and_log_complaints_aggregated_by_source(complaints):
     """Build formatted Slack message for posting to the API, aggregating
     complaints by source to reduce the number of blocks."""