Merge pull request #1211 from cmu-delphi/produce-microdata-raceeth

Support upload of raceeth microdata in Makefile

Author: krivard

facebook/Makefile

@@ -13,6 +13,7 @@ QUALTRICS=$(shell $(PYTHON) -m delphi_utils get input_dir)
 WEIGHTS=$(shell $(PYTHON) -m delphi_utils get weights_in_dir)
 CIDS=$(shell $(PYTHON) -m delphi_utils get weights_out_dir)
 INDIVIDUAL=$(shell $(PYTHON) -m delphi_utils get individual_dir)
+INDIVIDUAL_RACEETH=$(shell $(PYTHON) -m delphi_utils get individual_raceeth_dir)
 ARCHIVE=$(shell $(PYTHON) -m delphi_utils get archive_dir)
 RECEIVING=$(shell $(PYTHON) -m delphi_utils get export_dir)
 FB_CC=$(shell $(PYTHON) -m delphi_utils get qualtrics.notify-bad-weights)
@@ -26,6 +27,7 @@ MAX_WEIGHTED=ls -1 $(WEIGHTS) | grep dap | tail -1 | sed 's/_.*//;s/-//g;'
 ANTIJOIN:="antijoin.cids.sorted.txt"
 CIDS_DEST:="fb-interchange/cmu_respondent_ids"
 INDIVID_DEST:="fb-public-results/"
+INDIVID_RACEETH_DEST:="protected-race-ethnicity-data/"
 RAW_DEST:="raw"
 
 # dry-run mode: generate all files, but do not post them anywhere, and disable all emails to outside parties.
@@ -52,11 +54,12 @@ tidy: receiving
 	cp params.json tidy/
 	mv $(RECEIVING)/*.csv tidy/$(RECEIVING)
 	mv $(INDIVIDUAL)/*.csv* tidy/$(INDIVIDUAL)
+	mv $(INDIVIDUAL_RACEETH)/*.csv* tidy/$(INDIVIDUAL_RACEETH)
 	tar -czf scratch/tidy-`date +"%Y-%m-%d-%H%M%S"`.tgz --exclude='tidy-*.tgz' tidy
 	mv scratch/*.tgz tidy/
 
 clean:
-	rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(CIDS)/*.csv
+	rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(INDIVIDUAL_RACEETH)/*.csv  $(CIDS)/*.csv
 
 clean-archive:
 	rm -f $(ARCHIVE)/*.Rds
@@ -98,6 +101,7 @@ params.json: $(TODAY)
 	PAT=`grep fb-survey params.json | awk 'BEGIN{FS="\""}{print $$2}' | sed 's/ /_/g;s/^/-e /'`; \
 	$(PYTHON) -m delphi_utils set \
 		debug false \
+		produce_individual_raceeth true \
 		end_date $(YESTERDAY) \
 		input <(find $(QUALTRICS) -maxdepth 1 -newer $< -type f -name "*.csv" | sort | grep $${PAT}  | tr '\n' ',' | sed 's_$(QUALTRICS)/__g;s/,$$//' ) \
 		parallel true \
@@ -136,7 +140,7 @@ run-R: $(CIDS)
 	grep "scheduled core" tmp ; \
 	[ "$$?" -eq 1 ]
 
-pipeline: scratch init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-done tidy
+pipeline: scratch init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-individual-raceeth post-done tidy
 	grep $(TODAY) params.json
 	[ -f $(YESTERDAY) ] && rm $(YESTERDAY) || true
 	touch $@
@@ -193,6 +197,19 @@ post-individual: $(TODAY) $(INDIVIDUAL)
 	echo "SUCCESS: $(DRY_MESSAGE)Posted `echo $${POST} | wc -w` microresponse files" >> $(MESSAGES)
 	touch $@
 
+post-individual-raceeth: $(TODAY) $(INDIVIDUAL_RACEETH)
+	POST=`find $(INDIVIDUAL_RACEETH) -maxdepth 1 -newer $(TODAY) -name "cvid_responses_*.csv"`; \
+	[ -n "$${POST}" ]; \
+	BATCH=""; \
+	for f in $${POST}; do \
+	    (grep token $$f; [[ $$? -eq 1 ]]); \
+	    gzip -f $$f; \
+	    BATCH="$${BATCH}put $${f}.gz ${INDIVID_RACEETH_DEST}\n"; \
+	done; \
+	$(SFTP_POST); \
+	echo "SUCCESS: $(DRY_MESSAGE)Posted `echo $${POST} | wc -w` race-ethnicity microresponse files" >> $(MESSAGES)
+	touch $@
+
 post-done: post-cids
 	touch $(YESTERDAY).done
 	BATCH="put $(YESTERDAY).done $(CIDS_DEST)\n"; \

facebook/params.json.production.template

@@ -7,6 +7,8 @@
   "end_date": "2020-08-28",
   "export_dir": "./receiving",
   "individual_dir": "./individual",
+  "individual_raceeth_dir": "./individual_raceeth",
+  "produce_individual_raceeth": false,
   "input": [
     "2020-08-29.2020-08-22.2020-08-29.Survey_of_COVID-Like_Illness_-_TODEPLOY_......_-_US_Expansion.csv",
     "2020-08-29.2020-08-22.2020-08-29.Survey_of_COVID-Like_Illness_-_TODEPLOY-_US_Expansion_-_With_Translations.csv"