Merge pull request #1460 from cmu-delphi/release/indicators_v0.2.20_utils_v0.2.9

Release covidcast-indicators 0.2.20

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.19
+current_version = 0.2.20
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

.github/workflows/python-ci.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, covid_act_now, doctor_visits, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot, usafacts]
+        packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, doctor_visits, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot, usafacts]
     defaults:
       run:
         working-directory: ${{ matrix.packages }}

Jenkinsfile

@@ -9,7 +9,7 @@
    - Keep in sync with '.github/workflows/python-ci.yml'.
    - TODO: #527 Get this list automatically from python-ci.yml at runtime.
  */
-def indicator_list = ["changehc", "claims_hosp", "covid_act_now", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts"]
+def indicator_list = ["changehc", "claims_hosp", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts"]
 def build_package = [:]
 def deploy_staging = [:]
 def deploy_production = [:]

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.8
+current_version = 0.2.9
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.2.8"
+__version__ = "0.2.9"

_delphi_utils_python/delphi_utils/validator/datafetcher.py

@@ -111,8 +111,10 @@ def get_geo_signal_combos(data_source):
     Cross references based on combinations reported available by COVIDcast metadata.
     """
     # Maps data_source name with what's in the API, lists used in case of multiple names
+    # pylint: disable=fixme
+    # TODO: Extract this mapping from meta response instead of hard-coding
+    # https://github.com/cmu-delphi/covidcast-indicators/issues/1457
     source_signal_mappings = {
-        'chng': ['chng-cli', 'chng-covid'],
         'indicator-combination': ['indicator-combination-cases-deaths'],
         'quidel': ['quidel-covid-ag'],
         'safegraph': ['safegraph-weekly']

_delphi_utils_python/setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.2.8",
+    version="0.2.9",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

_delphi_utils_python/tests/validator/test_datafetcher.py

@@ -21,6 +21,9 @@ def test_make_date_filter(self):
         assert not date_filter(FILENAME_REGEX.match("20200620_a_b.csv"))
         assert not date_filter(FILENAME_REGEX.match("202006_a_b.csv"))
 
+    # pylint: disable=fixme
+    # TODO: mock out the advanced meta endpoint /covidcast/meta as well
+    # https://github.com/cmu-delphi/covidcast-indicators/issues/1456
     @mock.patch("covidcast.metadata")
     def test_get_geo_signal_combos(self, mock_metadata):
         """Test that the geo signal combos are correctly pulled from the covidcast metadata."""

ansible/templates/sir_complainsalot-params-prod.json.j2

@@ -102,10 +102,6 @@
       "max_age":16,
       "maintainers": []
     },
-    "covid-act-now": {
-      "max_age":9,
-      "maintainers": []
-    },
     "hhs": {
       "max_age":8,
       "maintainers": []

facebook/delphiFacebook/R/contingency_indicators.R

@@ -73,13 +73,17 @@ get_aggs <- function() {
     ## Mask wearing and distancing
     "pct_wearing_mask_5d", "c_mask_often", compute_binary, jeffreys_binary,
     "pct_wearing_mask_7d", "c_mask_often_7d", compute_binary, jeffreys_binary,
+    "pct_others_masked", "c_others_masked", compute_binary, jeffreys_binary,
     "pct_others_masked_public", "c_others_masked_public", compute_binary, jeffreys_binary,
     "pct_others_distanced_public", "c_others_distanced_public", compute_binary, jeffreys_binary,
 
     "pct_cli", "hh_p_cli", compute_household_binary, jeffreys_count,
     "pct_ili", "hh_p_ili", compute_household_binary, jeffreys_count,
     "pct_hh_cmnty_cli", "hh_community_yes", compute_binary, jeffreys_binary,
+    "pct_nohh_cmnty_cli", "community_yes", compute_binary, jeffreys_binary,
     "pct_direct_contact", "c_direct_contact", compute_binary, jeffreys_binary,
+    "pct_avoid_contact", "c_avoid_contact", compute_binary, jeffreys_binary,
+    "pct_avoid_contact_7d", "c_avoid_contact_7d", compute_binary, jeffreys_binary,
 
     # symptoms
     "pct_symp_fever", "symp_fever", compute_binary, jeffreys_binary,
@@ -303,8 +307,6 @@ get_aggs <- function() {
     "pct_race_treated_fairly_healthcare", "b_race_treated_fairly_healthcare", compute_binary, jeffreys_binary,
     "pct_delayed_care_cost", "b_delayed_care_cost", compute_binary, jeffreys_binary,
 
-    "pct_worried_catch_covid", "mh_worried_catch_covid", compute_binary, jeffreys_binary,
-    
     # topics want to learn about
     "pct_want_info_covid_treatment", "i_want_info_covid_treatment", compute_binary, jeffreys_binary,
     "pct_want_info_vaccine_access", "i_want_info_vaccine_access", compute_binary, jeffreys_binary,
@@ -355,6 +357,8 @@ get_aggs <- function() {
     "pct_anxious_7d", "mh_anxious_7d", compute_binary, jeffreys_binary,
     "pct_depressed_7d", "mh_depressed_7d", compute_binary, jeffreys_binary,
     "pct_felt_isolated_7d", "mh_isolated_7d", compute_binary, jeffreys_binary,
+    # wave 11
+    "pct_worried_catch_covid", "mh_worried_catch_covid", compute_binary, jeffreys_binary,
 
     # travel outside state
     # pre-wave 10
@@ -420,7 +424,11 @@ get_aggs <- function() {
     "pct_vaccine_barrier_type_tried", "v_vaccine_barrier_type_tried", compute_binary, jeffreys_binary,
     "pct_vaccine_barrier_none_tried", "v_vaccine_barrier_none_tried", compute_binary, jeffreys_binary,
     "pct_vaccine_barrier_appointment_location_tried", "v_vaccine_barrier_appointment_location_tried", compute_binary, jeffreys_binary,
-    "pct_vaccine_barrier_other_tried", "v_vaccine_barrier_other_tried", compute_binary, jeffreys_binary
+    "pct_vaccine_barrier_other_tried", "v_vaccine_barrier_other_tried", compute_binary, jeffreys_binary,
+
+    # schooling
+    "pct_inperson_school_fulltime", "s_inperson_school_fulltime", compute_binary, jeffreys_binary,
+    "pct_inperson_school_parttime", "s_inperson_school_parttime", compute_binary, jeffreys_binary
   )
 
   aggs <- create_aggs_product(regions, groups, indicators)

facebook/delphiFacebook/R/contingency_variables.R

@@ -1040,6 +1040,36 @@ code_behaviors <- function(input_data, wave) {
   } else {
     input_data$c_direct_contact <- NA
   }
+
+  # avoid_contact
+  # Percentage of respondents that have reported having intentionally avoided
+  # contact with other people all or most of the time
+  # 1 = all of the time, 2 = most of the time, 3 = some of the time,
+  # 4 = none of the time
+  if ("C7" %in% names(input_data)) {
+    input_data$c_avoid_contact <- case_when(
+      input_data$C7 %in% c(1, 2) ~ 1,
+      input_data$C7 %in% c(3, 4) ~ 0,
+      TRUE ~ NA_real_
+    )
+  } else {
+    input_data$c_avoid_contact <- NA_real_
+  }
+
+  # avoid_contact_7d
+  # Percentage of respondents that have reported having intentionally avoided
+  # contact with other people all or most of the time in the last 7 days
+  # 1 = all of the time, 2 = most of the time, 3 = some of the time,
+  # 4 = a little of the time, 5 = none of the time
+  if ("C7a" %in% names(input_data)) {
+    input_data$c_avoid_contact_7d <- case_when(
+      input_data$C7a %in% c(1, 2) ~ 1,
+      input_data$C7a %in% c(3, 4, 5) ~ 0,
+      TRUE ~ NA_real_
+    )
+  } else {
+    input_data$c_avoid_contact_7d <- NA_real_
+  }
 
   return(input_data)
 }

facebook/delphiFacebook/R/variables.R

@@ -14,7 +14,11 @@
 #' @return list of same length, each entry of which is a vector of selected
 #'   options
 split_options <- function(column) {
-  return(strsplit(column, ",", fixed = TRUE))
+  if ( any(!is.na(column)) ) {
+    return(strsplit(column, ",", fixed = TRUE))
+  } else {
+    return(rep(list(NA_character_), length(column)))
+  }
 }
 
 #' Test if a specific selection is selected
@@ -648,16 +652,8 @@ code_vaccines <- function(input_data, wave) {
   if ( all(c("V15a", "V15b") %in% names(input_data)) ) {
     # introduced in Wave 11
     vaccine_barriers <- coalesce(input_data$V15a, input_data$V15b)
-
-    # If the entire column is NA, ifelse() results in a logical vector, not a
-    # character vector, which confuses split_options; since the result should be
-    # NA anyway
-    vaccine_barriers <- as.character(
-      ifelse(vaccine_barriers == "13", NA_character_, vaccine_barriers)
-    )
-    if (any(!is.na(vaccine_barriers))) {
-      vaccine_barriers <- split_options(vaccine_barriers)
-    }
+    vaccine_barriers <- ifelse(vaccine_barriers == "13", NA_character_, vaccine_barriers)
+    vaccine_barriers <- split_options(vaccine_barriers)
 
     input_data$v_vaccine_barrier_eligible <- is_selected(vaccine_barriers, "1")
     input_data$v_vaccine_barrier_no_appointments <- is_selected(vaccine_barriers, "2")
@@ -677,7 +673,7 @@ code_vaccines <- function(input_data, wave) {
   } else if ( all(c("V15c", "V15b") %in% names(input_data)) ) {
     # V15c introduced in Wave 12, replacing V15a with clarified wording.
     vaccine_barriers <- coalesce(input_data$V15c, input_data$V15b)
-    vaccine_barriers <- ifelse(vaccine_barriers == "13", NA, vaccine_barriers)
+    vaccine_barriers <- ifelse(vaccine_barriers == "13", NA_character_, vaccine_barriers)
     vaccine_barriers <- split_options(vaccine_barriers)
 
     input_data$v_vaccine_barrier_eligible <- is_selected(vaccine_barriers, "1")
@@ -769,15 +765,8 @@ code_vaccines <- function(input_data, wave) {
 
   if ( "V15b" %in% names(input_data) ) {
     # introduced in Wave 11
-    # If the entire column is NA, ifelse() results in a logical vector, not a
-    # character vector, which confuses split_options; since the result should be
-    # NA anyway
-    vaccine_barriers <- as.character(
-      ifelse(input_data$V15b == "13", NA, input_data$V15b)
-    )
-    if (any(!is.na(vaccine_barriers))) {
-      vaccine_barriers <- split_options(vaccine_barriers)
-    }
+    vaccine_barriers <- ifelse(input_data$V15b == "13", NA_character_, input_data$V15b)
+    vaccine_barriers <- split_options(vaccine_barriers)
 
     input_data$v_vaccine_barrier_eligible_tried <- is_selected(vaccine_barriers, "1")
     input_data$v_vaccine_barrier_no_appointments_tried <- is_selected(vaccine_barriers, "2")

facebook/delphiFacebook/src/RcppExports.cpp

@@ -5,6 +5,11 @@
 
 using namespace Rcpp;
 
+#ifdef RCPP_USE_GLOBAL_ROSTREAM
+Rcpp::Rostream<true>&  Rcpp::Rcout = Rcpp::Rcpp_cout_get();
+Rcpp::Rostream<false>& Rcpp::Rcerr = Rcpp::Rcpp_cerr_get();
+#endif
+
 // is_selected_cpp
 LogicalVector is_selected_cpp(List responses, String target);
 RcppExport SEXP _delphiFacebook_is_selected_cpp(SEXP responsesSEXP, SEXP targetSEXP) {

facebook/delphiFacebook/unit-tests/testthat/test-variables.R

@@ -2,6 +2,22 @@ library(testthat)
 
 context("Testing response coding")
 
+test_that("split_options splits correctly", {
+  expect_equal(split_options(c("1", "", "1,2")),
+               list(c("1"), character(0), c("1", "2")))
+  
+  # Input logical vector
+  expect_equal(split_options(c(NA, NA, NA)),
+               list(NA_character_, NA_character_, NA_character_))
+  
+  # Input character vector
+  expect_equal(split_options(c(NA_character_, NA_character_, NA_character_)),
+               list(NA_character_, NA_character_, NA_character_))
+  
+  expect_equal(split_options(c("", NA_character_, NA)),
+               list(character(0), NA_character_, NA_character_))
+})
+
 test_that("is_selected handles selections correctly", {
   expect_equal(is_selected(split_options(c("1", "", "1,2")), "1"),
                c(TRUE, NA, TRUE))
@@ -18,6 +34,18 @@ test_that("is_selected handles selections correctly", {
   expect_equal(is_selected(split_options(c("4,54", "3,6,2,54", "5,4,45")),
                            "54"),
                c(TRUE, TRUE, FALSE))
+  
+  expect_equal(is_selected(c(NA, NA, NA), "14"),
+               c(NA, NA, NA))
+  
+  expect_equal(is_selected(c(NA_character_, NA_character_, NA_character_), "14"),
+               c(NA, NA, NA))
+  
+  expect_equal(is_selected(list(NA, NA, NA), "14"),
+               c(NA, NA, NA))
+  
+  expect_equal(is_selected(list(NA_character_, NA_character_, NA_character_), "14"),
+               c(NA, NA, NA))
 })
 
 test_that("activities items correctly coded", {

facebook/micro/monthly-archive.sh

@@ -350,7 +350,7 @@ add_qdf_to_codebook <- function(qdf,
   } else {
     codebook <- read_csv(path_to_codebook, col_types = cols(
       .default = col_character(),
-      wave = col_integer(),
+      wave = col_double(),
       variable = col_character(),
       replaces = col_character(),
       description = col_character(),
@@ -399,7 +399,7 @@ add_static_fields <- function(codebook,
   codebook <- bind_rows(codebook, static_fields) %>% 
     filter(!(variable == "module" & wave < 11), # module field is only available for wave >= 11
            !(variable %in% c("wave", "UserLanguage", "fips") & wave < 4), # wave, UserLangauge, and fips fields are only available for wave >= 4
-           !(variable == "w12_treatment" & wave == 12.5), # experimental arm field is only available for wave == 12.5
+           !(variable == "w12_treatment" & wave != 12.5), # experimental arm field is only available for wave == 12.5
            variable != "Random_Number"
     )
 
@@ -438,7 +438,7 @@ get_static_fields <- function(wave,
 add_qsf_to_codebook <- function(path_to_qsf, path_to_codebook) {
   qdf <- process_qsf(path_to_qsf)
   codebook <- add_qdf_to_codebook(qdf, path_to_codebook)
-  write_excel_csv(codebook, path_to_codebook)
+  write_excel_csv(codebook, path_to_codebook, quote="needed")
 }
 
 

facebook/qsf-tools/generate-codebook.R

@@ -101,10 +101,6 @@
       "max_age":16,
       "maintainers": []
     },
-    "covid-act-now": {
-      "max_age":9,
-      "maintainers": []
-    },
     "hhs": {
       "max_age":8,
       "maintainers": []