Merge pull request #1109 from cmu-delphi/fb-package-codebook-feedback

Improve on initial version of survey codebook and incorporate feedback

Author: krivard

facebook/qsf-tools/README.md

@@ -20,11 +20,12 @@ questions. These mapping files are created manually and need to be updated for
 every new survey wave.
 
 * `item_replacement_map.csv`: Lists in-survey name of an `new_item` and the
-  in-survey name of the `old_item` it replaces. `new_item` should be the name
-  of a single item and be unique, but the `old_item` column has no formatting
-  requirements. It can hold a list of items, if the corresponding new survey
-  item is replacing multiple old questions, and a given item name can appear
-  in multiple rows of the `old_item` field.
+  in-survey name(s) of the `old_item`(s) it replaces. `new_item` should be the
+  name of a single item and be unique; the `old_item` column should be a
+  string. However, `old_item` has no other formatting requirements. For
+  example, it can list several item names (e.g. "A1, A2"), if the
+  corresponding new survey item is replacing multiple old questions. A given
+  item name can also appear in multiple rows of the `old_item` field.
 * `item_shortquestion_map.csv`: Lists in-survey name of an `item` and a short
   description of the contents of the question. `item` should be the name of a
   single item and be unique, but the `description` column has no formatting
@@ -83,4 +84,4 @@ which can contain any subset of the following fields:
 The meaning of "Answers" and "Choices" differs for matrix vs non-matrix
 items. "Choices" list the vertical components -- subquestions for matrix
 items and answer choices for non-matrix items. "Answers" list the answer
-choices for matrix items and are missing for non-matrix items.
+choices for matrix items and are missing for non-matrix items.

facebook/qsf-tools/generate-codebook.R

@@ -9,6 +9,7 @@
 suppressPackageStartupMessages({
   library(tidyverse)
   library(jsonlite)
+  library(rjson)
   library(stringr)
   library(gsubfn)
   source("qsf-utils.R")
@@ -265,7 +266,24 @@ process_qsf <- function(path_to_qsf,
                              NA_character_),
            wave = get_wave(path_to_qsf)
     ) %>% 
-    select(wave, variable, replaces, description, question, matrix_subquestion, type, display_logic, response_option_randomization, group_of_respondents_item_was_shown_to)
+    select(wave,
+           variable,
+           replaces,
+           description,
+           question,
+           matrix_subquestion,
+           choices,
+           type,
+           display_logic,
+           response_option_randomization,
+           group_of_respondents_item_was_shown_to)
+  
+  # Format choices as json string
+  qdf$choices <- map(qdf$choices, function(x) {
+    if (is_empty(x)) { NA }
+    else { toJSON(x) }
+  }) %>%
+    unlist()
 
   # add free text response options
   other_text_items <- qdf %>%
@@ -276,6 +294,7 @@ process_qsf <- function(path_to_qsf,
            description = paste0(description, " other text")
     )
   qdf <- rbind(qdf, other_text_items)
+  qdf$choices[qdf$type == "Text"] <- NA
 
   # Quality checks
   stopifnot(length(qdf$variable) == length(unique(qdf$variable)))
@@ -388,7 +407,7 @@ get_static_fields <- function(wave,
 add_qsf_to_codebook <- function(path_to_qsf, path_to_codebook) {
   qdf <- process_qsf(path_to_qsf)
   codebook <- add_qdf_to_codebook(qdf, path_to_codebook)
-  write_csv(codebook, path_to_codebook)
+  write_excel_csv(codebook, path_to_codebook)
 }
 
 

facebook/qsf-tools/static/item_replacement_map.csv

@@ -22,6 +22,7 @@ C15,Q36
 C13b,C13
 C13c,C13a
 C14a,C14
+C17,C2
 C17a,C17
 V2a,V2
 V3a,V3
@@ -32,4 +33,5 @@ V4a_4,V4_4
 V4a_5,V4_5
 V11a,V11
 V12a,V12
-C7a,C7
+C7a,C7
+B10c,B10a

facebook/qsf-tools/static/static_microdata_fields.csv

@@ -1,6 +1,6 @@
 variable,replaces,description,question,matrix_subquestion,type,response_option_randomization
-StartDatetime,NA,"survey start timestamp",NA,NA,NA,NA
-EndDatetime,NA,"survey end timestamp",NA,NA,NA,NA
+StartDatetime,NA,"survey start timestamp in Pacific time (UTC-7)",NA,NA,NA,NA
+EndDatetime,NA,"survey end timestamp in Pacific time (UTC-7)",NA,NA,NA,NA
 wave,NA,"survey version",NA,NA,NA,NA
 UserLanguage,NA,"survey language",NA,NA,NA,NA
 fips,NA,"county FIPS code",NA,NA,NA,NA