Merge pull request #1245 from cmu-delphi/release/indicators_v0.1.16_utils_v0.1.12

Release covidcast-indicators 0.1.16

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.15
+current_version = 0.1.16
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

ansible/templates/facebook-params-prod.json.j2

@@ -4,6 +4,10 @@
   "backfill_days": 4,
   "cache_dir": "./cache",
   "debug": false,
+  "delphi_survey_email_user": "{{ fb_survey_email_user }}",
+  "delphi_survey_sftp_password": "{{ fb_survey_sftp_password }}",
+  "delphi_survey_sftp_user": "{{ fb_survey_sftp_user }}",
+  "sftp_options": "-o 'BatchMode=no' -o 'StrictHostKeyChecking=no' -o 'PubkeyAuthentication=no' -o 'PreferredAuthentications=password'",
   "end_date": "2021-08-16",
   "export_dir": "./receiving",
   "individual_dir": "./individual",

ansible/vars.yaml

@@ -10,22 +10,32 @@ python_version: "3.8.2"
 pyenv_python_path: "/home/{{ runtime_user }}/.pyenv/versions/{{ python_version }}/bin/python"
 
 # Indicators variables
+# Google Health
 google_health_api_key: "{{ vault_google_health_api_key }}"
 delphi_aws_access_key_id: "{{ vault_delphi_aws_access_key_id }}"
 delphi_aws_secret_access_key: "{{ vault_delphi_aws_secret_access_key }}"
+# SafeGraph
 safegraph_aws_access_key_id: "{{ vault_safegraph_aws_access_key_id }}"
 safegraph_aws_secret_access_key: "{{ vault_safegraph_aws_secret_access_key }}"
+# Quidel
 quidel_aws_access_key_id: "{{ vault_quidel_aws_access_key_id }}"
 quidel_aws_secret_access_key: "{{ vault_quidel_aws_secret_access_key }}"
 quidel_aws_bucket_name: 'delphi-quidel-data'
+# Change Healthcare
 changehc_sftp_host: "{{ vault_changehc_sftp_host }}"
 changehc_sftp_port: "{{ vault_changehc_sftp_port }}"
 changehc_sftp_user: "{{ vault_changehc_sftp_user }}"
 changehc_sftp_password: "{{ vault_changehc_sftp_password }}"
+# NCHS
 nchs_mortality_token: "{{ vault_nchs_mortality_token }}"
+# SirCAL
 sir_complainsalot_slack_token: "{{ vault_sir_complainsalot_slack_token }}"
+# Survey
 fb_qualtrics_token: "{{ vault_fb_qualtrics_token }}"
 fb_notify_weights_email_list: "{{ vault_fb_notify_weights_email_list }}"
+fb_survey_email_user: "{{ vault_fb_survey_email_user }}"
+fb_survey_sftp_password: "{{ vault_fb_survey_sftp_password }}"
+fb_survey_sftp_user: "{{ vault_fb_survey_sftp_user }}"
 ## Google Symptoms
 google_symptoms_account_type: "{{ vault_google_symptoms_account_type }}"
 google_symptoms_project_id: "{{ vault_google_symptoms_project_id }}"

ansible/vault.yaml

@@ -0,0 +1,32 @@
+FROM rocker/tidyverse:latest
+
+# use delphi's timezome
+RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
+
+RUN install2.r --error \
+    jsonlite \
+    stringr \
+    stringi \
+    data.table \
+    roxygen2 \
+    Rcpp \
+    BH
+
+RUN R --no-restore --no-save -e 'devtools::install_version("readr", version="1.4.0")'
+RUN apt-get update && apt-get install -qq -y \
+    python3-venv \
+    sshpass \
+    openssh-server \
+    ssmtp
+
+ADD ./delphiFacebook /facebook/delphiFacebook/
+ADD ./static /facebook/static/
+ADD ./Makefile /facebook/Makefile
+ADD ./run.R /facebook/run.R
+ADD ./monthly-files.R /facebook/monthly-files.R
+ADD ./contingency_tables.R /facebook/contingency_tables.R
+ADD ./contingency-combine.R /facebook/contingency-combine.R
+ADD ./ssmtp.conf /etc/ssmtp/ssmtp.conf
+WORKDIR /facebook/
+RUN make lib
+RUN make install

facebook/Dockerfile

@@ -1,23 +1,12 @@
 SHELL:=/bin/bash --rcfile bash-init.sh
 
-# dry-run mode: generate all files, but do not post them anywhere, and disable all emails to outside parties.
-DRY:=yes
-ifeq ($(DRY),yes)
-	EMAIL_SEND:=echo -e "Would send mail: echo -e \"Subject: $${SUBJECT}\n\n$${MSG}\" | sendmail [email protected]"
-	SFTP_POST:=echo -e "Would run: sftp -b <(echo -e \"\$${BATCH}\") -P 2222 [email protected]\n$${BATCH}"
-	DRY_MESSAGE:="[DRY-RUN] "
-else
-	EMAIL_SEND:=echo -e "Subject: $${SUBJECT}\n\n$${MSG}" | sendmail [email protected]
-	SFTP_POST:=sftp -b <(echo -e "$${BATCH}") -P 2222 [email protected]
-endif
-
 EPOCH:=2020-04-06
 TODAY:=$(shell date +"%Y-%m-%d")
 YESTERDAY:=$(shell date --date "$(TODAY) -1 day" +"%Y-%m-%d")
 ONEWEEK:=$(shell date --date "$(TODAY) -7 day" +"%Y-%m-%d")
 THREEWEEK:=$(shell date --date "$(TODAY) -21 day" +"%Y-%m-%d")
 
-MESSAGES:=$(TODAY).messages
+MESSAGES:="messages/$(TODAY).messages"
 
 PYTHON:=env/bin/python
 QUALTRICS=$(shell $(PYTHON) -m delphi_utils get input_dir)
@@ -27,6 +16,10 @@ INDIVIDUAL=$(shell $(PYTHON) -m delphi_utils get individual_dir)
 ARCHIVE=$(shell $(PYTHON) -m delphi_utils get archive_dir)
 RECEIVING=$(shell $(PYTHON) -m delphi_utils get export_dir)
 FB_CC=$(shell $(PYTHON) -m delphi_utils get qualtrics.notify-bad-weights)
+DELPHI_SURVEY_SFTP_PASSWORD=$(shell $(PYTHON) -m delphi_utils get delphi_survey_sftp_password)
+DELPHI_SURVEY_SFTP_USER=$(shell $(PYTHON) -m delphi_utils get delphi_survey_sftp_user)
+DELPHI_SURVEY_EMAIL_USER=$(shell $(PYTHON) -m delphi_utils get delphi_survey_email_user)
+SFTP_OPTIONS=$(shell $(PYTHON) -m delphi_utils get sftp_options)
 
 MAX_WEIGHTED=ls -1 $(WEIGHTS) | grep dap | tail -1 | sed 's/_.*//;s/-//g;'
 
@@ -35,16 +28,34 @@ CIDS_DEST:="fb-interchange/cmu_respondent_ids"
 INDIVID_DEST:="fb-public-results/"
 RAW_DEST:="raw"
 
+# dry-run mode: generate all files, but do not post them anywhere, and disable all emails to outside parties.
+DRY:=yes
+ifeq ($(DRY),yes)
+	EMAIL_SEND:=echo -e "Would send mail: echo -e \"Subject: $${SUBJECT}\n\n$${MSG}\" | sendmail $(DELPHI_SURVEY_EMAIL_USER)"
+	SFTP_POST:=echo -e "Would run: sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e \"\$${BATCH}\") -P 2222 $(DELPHI_SURVEY_SFTP_USER)\n$${BATCH}"
+	DRY_MESSAGE:="[DRY-RUN] "
+else
+	EMAIL_SEND:=echo -e "Subject: $${SUBJECT}\n\n$${MSG}" | sendmail $(DELPHI_SURVEY_EMAIL_USER)
+	SFTP_POST:=sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e "$${BATCH}") -P 2222 $(DELPHI_SURVEY_SFTP_USER)
+endif
+
 default:
 	@echo No default implemented yet
 
+scratch:
+	mkdir scratch
+	rm -rf scratch/*
+
 tidy: receiving
-	rm -rf tidy
+	rm -rf tidy/$(RECEIVING)
+	rm -rf tidy/$(INDIVIDUAL)
+	rm -f tidy/params.json
 	mkdir -p tidy tidy/$(RECEIVING) tidy/$(INDIVIDUAL)
 	cp params.json tidy/
 	mv $(RECEIVING)/*.csv tidy/$(RECEIVING)
 	mv $(INDIVIDUAL)/*.csv* tidy/$(INDIVIDUAL)
-	tar -czf tidy-`date +"%Y-%m-%d-%H%M%S"`.tgz tidy
+	tar -czf scratch/tidy-`date +"%Y-%m-%d-%H%M%S"`.tgz tidy
+	mv scratch/*.tgz tidy/
 
 clean:
 	rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(CIDS)/*.csv
@@ -55,7 +66,8 @@ clean-archive:
 install-python:
 	python3.8 -m venv env
 	source env/bin/activate && \
-	pip install -e ../_delphi_utils_python && \
+	pip install wheel && \
+	pip install delphi_utils && \
 	pip install -e delphiFacebook/python
 
 install-R:dev
@@ -67,7 +79,6 @@ $(CIDS):
 
 init-qualtrics:
 	grep '"token": "..*"' params.json
-	git show -s --oneline HEAD >> $(MESSAGES)
 
 $(TODAY):
 	[ -f $(QUALTRICS) ] || mkdir -p $(QUALTRICS)
@@ -99,13 +110,13 @@ $(WEIGHTS): $(TODAY)
 	[ -f $(WEIGHTS) ] || mkdir -p $(WEIGHTS)
 	cd "$(WEIGHTS)"; \
 	BATCH="cd fb-interchange/cmu_respondent_weights\nls -1"; \
-	NEW=`LC_ALL=C comm -23 <(sftp -b <(echo -e "$${BATCH}") -P 2222 [email protected] | grep "^202" | LC_ALL=C sort) <(ls -1 | LC_ALL=C sort)`; \
+	NEW=`LC_ALL=C comm -23 <(sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e "$${BATCH}") -P 2222 $(DELPHI_SURVEY_SFTP_USER) | grep "^202" | LC_ALL=C sort) <(ls -1 | LC_ALL=C sort)`; \
 	echo "New weights files:"; \
 	echo $${NEW}; \
 	for f in $${NEW}; do \
 	  BATCH="$${BATCH}\nget $$f"; \
 	done; \
-	sftp -b <(echo -e "$${BATCH}") -P 2222 [email protected] || exit 90; \
+	sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e "$${BATCH}") -P 2222 $(DELPHI_SURVEY_SFTP_USER) || exit 90; \
 	cd -; \
 	MAX_WEIGHTED=`$(MAX_WEIGHTED)`; \
 	EXPECTED_MAX_WEIGHTED=`date --date='$(TODAY) -3 day' +'%Y%m%d'`; \
@@ -129,11 +140,12 @@ run-R: $(CIDS)
 	grep "scheduled core" tmp ; \
 	[ "$$?" -eq 1 ]
 
-pipeline: init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-done tidy
+pipeline: scratch init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-done tidy
 	grep $(TODAY) params.json
 	[ -f $(YESTERDAY) ] && rm $(YESTERDAY) || true
 	touch $@
 	echo "SUCCESS: $(DRY_MESSAGE)pipeline complete" >> $(MESSAGES)
+	chmod o+w $(MESSAGES)
 
 coverage:
 	Rscript -e 'covr::package_coverage("delphiFacebook")'

facebook/Makefile

@@ -13,7 +13,7 @@ Depends:
 Imports:
     stats,
     rlang,
-    readr,
+    readr (<= 1.4.0),
     dplyr,
     stringi,
     jsonlite,

facebook/delphiFacebook/DESCRIPTION

@@ -92,6 +92,17 @@ process_qsf <- function(path_to_qsf,
   choices <- displayed_questions %>% 
     map(~ .x$Payload$Choices) %>% 
     map(~ map(.x, "Display"))
+  recode_map <- displayed_questions %>% 
+    map(~ .x$Payload$RecodeValues)
+  
+  # Recode response options if overriding Qualtrics auto-assigned coding.
+  ii_recode <- recode_map %>%
+    map(~ !is.null(.x)) %>% 
+    unlist() %>% 
+    which()
+  choices[ii_recode] <- map2(.x=choices[ii_recode], .y=recode_map[ii_recode],
+                             ~ setNames(.x, .y[names(.x)])
+  )
 
   # derive from choices where users can fill in "Other" response with free text
   other_text_option <- displayed_questions %>% 
@@ -158,11 +169,25 @@ process_qsf <- function(path_to_qsf,
       map(.x, "Operator"),
       map(.x, "RightOperand")
     )) %>% 
+    # Remove empty logic
     map(~ gsub("  Is  ", "", .x)) %>% 
     map(~ gsub("NULL NULL Is NULL NULL", "", .x)) %>%
     map(~ gsub(" ?NULL ?", "", .x)) %>%
+    # Remove QID flag
     map(~ gsub("q://", "", .x)) %>%
+    # Recode choice numbers
+    map(~ gsubfn("(QID[0-9]+)(/SelectableChoice/)([0-9]+)", function(qid, selectable_text, option_code) {
+      curr_map <- recode_map[qids == qid][[1]]
+      
+      if ( !is.null(curr_map) ) {
+        option_code <- curr_map[names(curr_map) == option_code]
+      }
+      
+      paste(c(qid, selectable_text, option_code), collapse="")
+    }, .x)) %>% 
+    # Replace QID with question number (A2, etc)
     map(~ gsubfn("(QID[0-9]+)", function(qid) {items[qids == qid]}, .x)) %>% 
+    # Collapse logic into a single string.
     map(~ paste(.x, collapse=" "))
 
   logic_type <- displayed_questions %>% 

facebook/qsf-tools/generate-codebook.R

@@ -0,0 +1,21 @@
+#
+# Config file for sSMTP sendmail
+#
+# The person who gets all mail for userids < 1000
+# Make this empty to disable rewriting.
+root=postmaster
+
+# The place where the mail goes. The actual machine name is required no 
+# MX records are consulted. Commonly mailhosts are named mail.domain.com
+mailhub=localhost
+
+# Where will the mail seem to come from?
+#rewriteDomain=
+
+# The full hostname
+hostname=delphi.cmu.edu
+
+# Are users allowed to set their own From: address?
+# YES - Allow the user to specify their own From: address
+# NO - Use the system generated From: address
+#FromLineOverride=YES