diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 224acb5f4..4b2a79370 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.19 +current_version = 0.2.20 commit = True message = chore: bump covidcast-indicators to {new_version} tag = False diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 47c6b2f16..19fdf56d5 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -16,7 +16,7 @@ jobs: if: github.event.pull_request.draft == false strategy: matrix: - packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, covid_act_now, doctor_visits, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot, usafacts] + packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, doctor_visits, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot, usafacts] defaults: run: working-directory: ${{ matrix.packages }} diff --git a/Jenkinsfile b/Jenkinsfile index 85c3bfd6f..a54087336 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,7 @@ - Keep in sync with '.github/workflows/python-ci.yml'. - TODO: #527 Get this list automatically from python-ci.yml at runtime. */ -def indicator_list = ["changehc", "claims_hosp", "covid_act_now", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts"] +def indicator_list = ["changehc", "claims_hosp", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts"] def build_package = [:] def deploy_staging = [:] def deploy_production = [:] diff --git a/_delphi_utils_python/.bumpversion.cfg b/_delphi_utils_python/.bumpversion.cfg index 222ae37ed..6a3eb41c5 100644 --- a/_delphi_utils_python/.bumpversion.cfg +++ b/_delphi_utils_python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.8 +current_version = 0.2.9 commit = True message = chore: bump delphi_utils to {new_version} tag = False diff --git a/_delphi_utils_python/delphi_utils/__init__.py b/_delphi_utils_python/delphi_utils/__init__.py index df6bb0402..8e5e8e86a 100644 --- a/_delphi_utils_python/delphi_utils/__init__.py +++ b/_delphi_utils_python/delphi_utils/__init__.py @@ -15,4 +15,4 @@ from .nancodes import Nans from .weekday import Weekday -__version__ = "0.2.8" +__version__ = "0.2.9" diff --git a/_delphi_utils_python/delphi_utils/validator/datafetcher.py b/_delphi_utils_python/delphi_utils/validator/datafetcher.py index 5a0cd67d6..17db7c4bb 100644 --- a/_delphi_utils_python/delphi_utils/validator/datafetcher.py +++ b/_delphi_utils_python/delphi_utils/validator/datafetcher.py @@ -111,8 +111,10 @@ def get_geo_signal_combos(data_source): Cross references based on combinations reported available by COVIDcast metadata. """ # Maps data_source name with what's in the API, lists used in case of multiple names + # pylint: disable=fixme + # TODO: Extract this mapping from meta response instead of hard-coding + # https://github.com/cmu-delphi/covidcast-indicators/issues/1457 source_signal_mappings = { - 'chng': ['chng-cli', 'chng-covid'], 'indicator-combination': ['indicator-combination-cases-deaths'], 'quidel': ['quidel-covid-ag'], 'safegraph': ['safegraph-weekly'] diff --git a/_delphi_utils_python/setup.py b/_delphi_utils_python/setup.py index 53c9a4b1c..064eb64b8 100644 --- a/_delphi_utils_python/setup.py +++ b/_delphi_utils_python/setup.py @@ -26,7 +26,7 @@ setup( name="delphi_utils", - version="0.2.8", + version="0.2.9", description="Shared Utility Functions for Indicators", long_description=long_description, long_description_content_type="text/markdown", diff --git a/_delphi_utils_python/tests/validator/test_datafetcher.py b/_delphi_utils_python/tests/validator/test_datafetcher.py index 524b34e83..9eb583e06 100644 --- a/_delphi_utils_python/tests/validator/test_datafetcher.py +++ b/_delphi_utils_python/tests/validator/test_datafetcher.py @@ -21,6 +21,9 @@ def test_make_date_filter(self): assert not date_filter(FILENAME_REGEX.match("20200620_a_b.csv")) assert not date_filter(FILENAME_REGEX.match("202006_a_b.csv")) + # pylint: disable=fixme + # TODO: mock out the advanced meta endpoint /covidcast/meta as well + # https://github.com/cmu-delphi/covidcast-indicators/issues/1456 @mock.patch("covidcast.metadata") def test_get_geo_signal_combos(self, mock_metadata): """Test that the geo signal combos are correctly pulled from the covidcast metadata.""" diff --git a/ansible/templates/sir_complainsalot-params-prod.json.j2 b/ansible/templates/sir_complainsalot-params-prod.json.j2 index 7a4dfae7b..94c2c8160 100644 --- a/ansible/templates/sir_complainsalot-params-prod.json.j2 +++ b/ansible/templates/sir_complainsalot-params-prod.json.j2 @@ -102,10 +102,6 @@ "max_age":16, "maintainers": [] }, - "covid-act-now": { - "max_age":9, - "maintainers": [] - }, "hhs": { "max_age":8, "maintainers": [] diff --git a/facebook/delphiFacebook/R/contingency_indicators.R b/facebook/delphiFacebook/R/contingency_indicators.R index f5f652a45..426b362d0 100644 --- a/facebook/delphiFacebook/R/contingency_indicators.R +++ b/facebook/delphiFacebook/R/contingency_indicators.R @@ -73,13 +73,17 @@ get_aggs <- function() { ## Mask wearing and distancing "pct_wearing_mask_5d", "c_mask_often", compute_binary, jeffreys_binary, "pct_wearing_mask_7d", "c_mask_often_7d", compute_binary, jeffreys_binary, + "pct_others_masked", "c_others_masked", compute_binary, jeffreys_binary, "pct_others_masked_public", "c_others_masked_public", compute_binary, jeffreys_binary, "pct_others_distanced_public", "c_others_distanced_public", compute_binary, jeffreys_binary, "pct_cli", "hh_p_cli", compute_household_binary, jeffreys_count, "pct_ili", "hh_p_ili", compute_household_binary, jeffreys_count, "pct_hh_cmnty_cli", "hh_community_yes", compute_binary, jeffreys_binary, + "pct_nohh_cmnty_cli", "community_yes", compute_binary, jeffreys_binary, "pct_direct_contact", "c_direct_contact", compute_binary, jeffreys_binary, + "pct_avoid_contact", "c_avoid_contact", compute_binary, jeffreys_binary, + "pct_avoid_contact_7d", "c_avoid_contact_7d", compute_binary, jeffreys_binary, # symptoms "pct_symp_fever", "symp_fever", compute_binary, jeffreys_binary, @@ -303,8 +307,6 @@ get_aggs <- function() { "pct_race_treated_fairly_healthcare", "b_race_treated_fairly_healthcare", compute_binary, jeffreys_binary, "pct_delayed_care_cost", "b_delayed_care_cost", compute_binary, jeffreys_binary, - "pct_worried_catch_covid", "mh_worried_catch_covid", compute_binary, jeffreys_binary, - # topics want to learn about "pct_want_info_covid_treatment", "i_want_info_covid_treatment", compute_binary, jeffreys_binary, "pct_want_info_vaccine_access", "i_want_info_vaccine_access", compute_binary, jeffreys_binary, @@ -355,6 +357,8 @@ get_aggs <- function() { "pct_anxious_7d", "mh_anxious_7d", compute_binary, jeffreys_binary, "pct_depressed_7d", "mh_depressed_7d", compute_binary, jeffreys_binary, "pct_felt_isolated_7d", "mh_isolated_7d", compute_binary, jeffreys_binary, + # wave 11 + "pct_worried_catch_covid", "mh_worried_catch_covid", compute_binary, jeffreys_binary, # travel outside state # pre-wave 10 @@ -420,7 +424,11 @@ get_aggs <- function() { "pct_vaccine_barrier_type_tried", "v_vaccine_barrier_type_tried", compute_binary, jeffreys_binary, "pct_vaccine_barrier_none_tried", "v_vaccine_barrier_none_tried", compute_binary, jeffreys_binary, "pct_vaccine_barrier_appointment_location_tried", "v_vaccine_barrier_appointment_location_tried", compute_binary, jeffreys_binary, - "pct_vaccine_barrier_other_tried", "v_vaccine_barrier_other_tried", compute_binary, jeffreys_binary + "pct_vaccine_barrier_other_tried", "v_vaccine_barrier_other_tried", compute_binary, jeffreys_binary, + + # schooling + "pct_inperson_school_fulltime", "s_inperson_school_fulltime", compute_binary, jeffreys_binary, + "pct_inperson_school_parttime", "s_inperson_school_parttime", compute_binary, jeffreys_binary ) aggs <- create_aggs_product(regions, groups, indicators) diff --git a/facebook/delphiFacebook/R/contingency_variables.R b/facebook/delphiFacebook/R/contingency_variables.R index 714cd4a09..93ce4a18e 100644 --- a/facebook/delphiFacebook/R/contingency_variables.R +++ b/facebook/delphiFacebook/R/contingency_variables.R @@ -1040,6 +1040,36 @@ code_behaviors <- function(input_data, wave) { } else { input_data$c_direct_contact <- NA } + + # avoid_contact + # Percentage of respondents that have reported having intentionally avoided + # contact with other people all or most of the time + # 1 = all of the time, 2 = most of the time, 3 = some of the time, + # 4 = none of the time + if ("C7" %in% names(input_data)) { + input_data$c_avoid_contact <- case_when( + input_data$C7 %in% c(1, 2) ~ 1, + input_data$C7 %in% c(3, 4) ~ 0, + TRUE ~ NA_real_ + ) + } else { + input_data$c_avoid_contact <- NA_real_ + } + + # avoid_contact_7d + # Percentage of respondents that have reported having intentionally avoided + # contact with other people all or most of the time in the last 7 days + # 1 = all of the time, 2 = most of the time, 3 = some of the time, + # 4 = a little of the time, 5 = none of the time + if ("C7a" %in% names(input_data)) { + input_data$c_avoid_contact_7d <- case_when( + input_data$C7a %in% c(1, 2) ~ 1, + input_data$C7a %in% c(3, 4, 5) ~ 0, + TRUE ~ NA_real_ + ) + } else { + input_data$c_avoid_contact_7d <- NA_real_ + } return(input_data) } diff --git a/facebook/delphiFacebook/R/variables.R b/facebook/delphiFacebook/R/variables.R index 6baa4ca39..b4461776d 100644 --- a/facebook/delphiFacebook/R/variables.R +++ b/facebook/delphiFacebook/R/variables.R @@ -14,7 +14,11 @@ #' @return list of same length, each entry of which is a vector of selected #' options split_options <- function(column) { - return(strsplit(column, ",", fixed = TRUE)) + if ( any(!is.na(column)) ) { + return(strsplit(column, ",", fixed = TRUE)) + } else { + return(rep(list(NA_character_), length(column))) + } } #' Test if a specific selection is selected @@ -648,16 +652,8 @@ code_vaccines <- function(input_data, wave) { if ( all(c("V15a", "V15b") %in% names(input_data)) ) { # introduced in Wave 11 vaccine_barriers <- coalesce(input_data$V15a, input_data$V15b) - - # If the entire column is NA, ifelse() results in a logical vector, not a - # character vector, which confuses split_options; since the result should be - # NA anyway - vaccine_barriers <- as.character( - ifelse(vaccine_barriers == "13", NA_character_, vaccine_barriers) - ) - if (any(!is.na(vaccine_barriers))) { - vaccine_barriers <- split_options(vaccine_barriers) - } + vaccine_barriers <- ifelse(vaccine_barriers == "13", NA_character_, vaccine_barriers) + vaccine_barriers <- split_options(vaccine_barriers) input_data$v_vaccine_barrier_eligible <- is_selected(vaccine_barriers, "1") input_data$v_vaccine_barrier_no_appointments <- is_selected(vaccine_barriers, "2") @@ -677,7 +673,7 @@ code_vaccines <- function(input_data, wave) { } else if ( all(c("V15c", "V15b") %in% names(input_data)) ) { # V15c introduced in Wave 12, replacing V15a with clarified wording. vaccine_barriers <- coalesce(input_data$V15c, input_data$V15b) - vaccine_barriers <- ifelse(vaccine_barriers == "13", NA, vaccine_barriers) + vaccine_barriers <- ifelse(vaccine_barriers == "13", NA_character_, vaccine_barriers) vaccine_barriers <- split_options(vaccine_barriers) input_data$v_vaccine_barrier_eligible <- is_selected(vaccine_barriers, "1") @@ -769,15 +765,8 @@ code_vaccines <- function(input_data, wave) { if ( "V15b" %in% names(input_data) ) { # introduced in Wave 11 - # If the entire column is NA, ifelse() results in a logical vector, not a - # character vector, which confuses split_options; since the result should be - # NA anyway - vaccine_barriers <- as.character( - ifelse(input_data$V15b == "13", NA, input_data$V15b) - ) - if (any(!is.na(vaccine_barriers))) { - vaccine_barriers <- split_options(vaccine_barriers) - } + vaccine_barriers <- ifelse(input_data$V15b == "13", NA_character_, input_data$V15b) + vaccine_barriers <- split_options(vaccine_barriers) input_data$v_vaccine_barrier_eligible_tried <- is_selected(vaccine_barriers, "1") input_data$v_vaccine_barrier_no_appointments_tried <- is_selected(vaccine_barriers, "2") diff --git a/facebook/delphiFacebook/src/RcppExports.cpp b/facebook/delphiFacebook/src/RcppExports.cpp index b0056334a..e21e5728d 100644 --- a/facebook/delphiFacebook/src/RcppExports.cpp +++ b/facebook/delphiFacebook/src/RcppExports.cpp @@ -5,6 +5,11 @@ using namespace Rcpp; +#ifdef RCPP_USE_GLOBAL_ROSTREAM +Rcpp::Rostream& Rcpp::Rcout = Rcpp::Rcpp_cout_get(); +Rcpp::Rostream& Rcpp::Rcerr = Rcpp::Rcpp_cerr_get(); +#endif + // is_selected_cpp LogicalVector is_selected_cpp(List responses, String target); RcppExport SEXP _delphiFacebook_is_selected_cpp(SEXP responsesSEXP, SEXP targetSEXP) { diff --git a/facebook/delphiFacebook/unit-tests/testthat/test-variables.R b/facebook/delphiFacebook/unit-tests/testthat/test-variables.R index bd894f49f..741a109a6 100644 --- a/facebook/delphiFacebook/unit-tests/testthat/test-variables.R +++ b/facebook/delphiFacebook/unit-tests/testthat/test-variables.R @@ -2,6 +2,22 @@ library(testthat) context("Testing response coding") +test_that("split_options splits correctly", { + expect_equal(split_options(c("1", "", "1,2")), + list(c("1"), character(0), c("1", "2"))) + + # Input logical vector + expect_equal(split_options(c(NA, NA, NA)), + list(NA_character_, NA_character_, NA_character_)) + + # Input character vector + expect_equal(split_options(c(NA_character_, NA_character_, NA_character_)), + list(NA_character_, NA_character_, NA_character_)) + + expect_equal(split_options(c("", NA_character_, NA)), + list(character(0), NA_character_, NA_character_)) +}) + test_that("is_selected handles selections correctly", { expect_equal(is_selected(split_options(c("1", "", "1,2")), "1"), c(TRUE, NA, TRUE)) @@ -18,6 +34,18 @@ test_that("is_selected handles selections correctly", { expect_equal(is_selected(split_options(c("4,54", "3,6,2,54", "5,4,45")), "54"), c(TRUE, TRUE, FALSE)) + + expect_equal(is_selected(c(NA, NA, NA), "14"), + c(NA, NA, NA)) + + expect_equal(is_selected(c(NA_character_, NA_character_, NA_character_), "14"), + c(NA, NA, NA)) + + expect_equal(is_selected(list(NA, NA, NA), "14"), + c(NA, NA, NA)) + + expect_equal(is_selected(list(NA_character_, NA_character_, NA_character_), "14"), + c(NA, NA, NA)) }) test_that("activities items correctly coded", { diff --git a/facebook/micro/monthly-archive.sh b/facebook/micro/monthly-archive.sh old mode 100644 new mode 100755 diff --git a/facebook/qsf-tools/generate-codebook.R b/facebook/qsf-tools/generate-codebook.R index 50eedbbee..ec3dda44d 100644 --- a/facebook/qsf-tools/generate-codebook.R +++ b/facebook/qsf-tools/generate-codebook.R @@ -350,7 +350,7 @@ add_qdf_to_codebook <- function(qdf, } else { codebook <- read_csv(path_to_codebook, col_types = cols( .default = col_character(), - wave = col_integer(), + wave = col_double(), variable = col_character(), replaces = col_character(), description = col_character(), @@ -399,7 +399,7 @@ add_static_fields <- function(codebook, codebook <- bind_rows(codebook, static_fields) %>% filter(!(variable == "module" & wave < 11), # module field is only available for wave >= 11 !(variable %in% c("wave", "UserLanguage", "fips") & wave < 4), # wave, UserLangauge, and fips fields are only available for wave >= 4 - !(variable == "w12_treatment" & wave == 12.5), # experimental arm field is only available for wave == 12.5 + !(variable == "w12_treatment" & wave != 12.5), # experimental arm field is only available for wave == 12.5 variable != "Random_Number" ) @@ -438,7 +438,7 @@ get_static_fields <- function(wave, add_qsf_to_codebook <- function(path_to_qsf, path_to_codebook) { qdf <- process_qsf(path_to_qsf) codebook <- add_qdf_to_codebook(qdf, path_to_codebook) - write_excel_csv(codebook, path_to_codebook) + write_excel_csv(codebook, path_to_codebook, quote="needed") } diff --git a/sir_complainsalot/params.json.template b/sir_complainsalot/params.json.template index a09254f7f..2df469f02 100644 --- a/sir_complainsalot/params.json.template +++ b/sir_complainsalot/params.json.template @@ -101,10 +101,6 @@ "max_age":16, "maintainers": [] }, - "covid-act-now": { - "max_age":9, - "maintainers": [] - }, "hhs": { "max_age":8, "maintainers": []