From 43183123a85040511181b34750993fb5312d728b Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Wed, 18 Aug 2021 12:16:28 -0400
Subject: [PATCH 1/2] support upload of raceeth microdata in Makefil

---
 facebook/Makefile | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/facebook/Makefile b/facebook/Makefile
index 5e0219ce2..b12f65093 100644
--- a/facebook/Makefile
+++ b/facebook/Makefile
@@ -24,6 +24,7 @@ QUALTRICS=$(shell $(PYTHON) -m delphi_utils get input_dir)
 WEIGHTS=$(shell $(PYTHON) -m delphi_utils get weights_in_dir)
 CIDS=$(shell $(PYTHON) -m delphi_utils get weights_out_dir)
 INDIVIDUAL=$(shell $(PYTHON) -m delphi_utils get individual_dir)
+INDIVIDUAL_RACEETH=$(shell $(PYTHON) -m delphi_utils get individual_raceeth_dir)
 ARCHIVE=$(shell $(PYTHON) -m delphi_utils get archive_dir)
 RECEIVING=$(shell $(PYTHON) -m delphi_utils get export_dir)
 FB_CC=$(shell $(PYTHON) -m delphi_utils get qualtrics.notify-bad-weights)
@@ -33,6 +34,7 @@ MAX_WEIGHTED=ls -1 $(WEIGHTS) | grep dap | tail -1 | sed 's/_.*//;s/-//g;'
 ANTIJOIN:="antijoin.cids.sorted.txt"
 CIDS_DEST:="fb-interchange/cmu_respondent_ids"
 INDIVID_DEST:="fb-public-results/"
+INDIVID_RACEETH_DEST:="protected-race-ethnicity-data/"
 RAW_DEST:="raw"
 
 default:
@@ -44,10 +46,11 @@ tidy: receiving
 	cp params.json tidy/
 	mv $(RECEIVING)/*.csv tidy/$(RECEIVING)
 	mv $(INDIVIDUAL)/*.csv* tidy/$(INDIVIDUAL)
+	mv $(INDIVIDUAL_RACEETH)/*.csv* tidy/$(INDIVIDUAL_RACEETH)
 	tar -czf tidy-`date +"%Y-%m-%d-%H%M%S"`.tgz tidy
 
 clean:
-	rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(CIDS)/*.csv
+	rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(INDIVIDUAL_RACEETH)/*.csv  $(CIDS)/*.csv
 
 clean-archive:
 	rm -f $(ARCHIVE)/*.Rds
@@ -129,7 +132,7 @@ run-R: $(CIDS)
 	grep "scheduled core" tmp ; \
 	[ "$$?" -eq 1 ]
 
-pipeline: init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-done tidy
+pipeline: init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-individual-raceeth post-done tidy
 	grep $(TODAY) params.json
 	[ -f $(YESTERDAY) ] && rm $(YESTERDAY) || true
 	touch $@
@@ -185,6 +188,19 @@ post-individual: $(TODAY) $(INDIVIDUAL)
 	echo "SUCCESS: $(DRY_MESSAGE)Posted `echo $${POST} | wc -w` microresponse files" >> $(MESSAGES)
 	touch $@
 
+post-individual-raceeth: $(TODAY) $(INDIVIDUAL_RACEETH)
+	POST=`find $(INDIVIDUAL_RACEETH) -maxdepth 1 -newer $(TODAY) -name "cvid_responses_*.csv"`; \
+	[ -n "$${POST}" ]; \
+	BATCH=""; \
+	for f in $${POST}; do \
+	    (grep token $$f; [[ $$? -eq 1 ]]); \
+	    gzip -f $$f; \
+	    BATCH="$${BATCH}put $${f}.gz ${INDIVID_RACEETH_DEST}\n"; \
+	done; \
+	$(SFTP_POST); \
+	echo "SUCCESS: $(DRY_MESSAGE)Posted `echo $${POST} | wc -w` race-ethnicity microresponse files" >> $(MESSAGES)
+	touch $@
+
 post-done: post-cids
 	touch $(YESTERDAY).done
 	BATCH="put $(YESTERDAY).done $(CIDS_DEST)\n"; \

From c7f63d456442440ab0771be1750dc5801b9b7ec9 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Wed, 18 Aug 2021 12:59:34 -0400
Subject: [PATCH 2/2] add new keys to params prod template; turn on raceeth
 microdata in make

---
 facebook/Makefile                        | 1 +
 facebook/params.json.production.template | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/facebook/Makefile b/facebook/Makefile
index b12f65093..df10d0fdf 100644
--- a/facebook/Makefile
+++ b/facebook/Makefile
@@ -92,6 +92,7 @@ params.json: $(TODAY)
 	PAT=`grep fb-survey params.json | awk 'BEGIN{FS="\""}{print $$2}' | sed 's/ /_/g;s/^/-e /'`; \
 	$(PYTHON) -m delphi_utils set \
 		debug false \
+		produce_individual_raceeth true \
 		end_date $(YESTERDAY) \
 		input <(find $(QUALTRICS) -maxdepth 1 -newer $< -type f -name "*.csv" | sort | grep $${PAT}  | tr '\n' ',' | sed 's_$(QUALTRICS)/__g;s/,$$//' ) \
 		parallel true \
diff --git a/facebook/params.json.production.template b/facebook/params.json.production.template
index 2756d80dc..79dc1aaab 100644
--- a/facebook/params.json.production.template
+++ b/facebook/params.json.production.template
@@ -7,6 +7,8 @@
   "end_date": "2020-08-28",
   "export_dir": "./receiving",
   "individual_dir": "./individual",
+  "individual_raceeth_dir": "./individual_raceeth",
+  "produce_individual_raceeth": false,
   "input": [
     "2020-08-29.2020-08-22.2020-08-29.Survey_of_COVID-Like_Illness_-_TODEPLOY_......_-_US_Expansion.csv",
     "2020-08-29.2020-08-22.2020-08-29.Survey_of_COVID-Like_Illness_-_TODEPLOY-_US_Expansion_-_With_Translations.csv"