diff --git a/facebook/Makefile b/facebook/Makefile index 3ed8824d8..967140d00 100644 --- a/facebook/Makefile +++ b/facebook/Makefile @@ -13,6 +13,7 @@ QUALTRICS=$(shell $(PYTHON) -m delphi_utils get input_dir) WEIGHTS=$(shell $(PYTHON) -m delphi_utils get weights_in_dir) CIDS=$(shell $(PYTHON) -m delphi_utils get weights_out_dir) INDIVIDUAL=$(shell $(PYTHON) -m delphi_utils get individual_dir) +INDIVIDUAL_RACEETH=$(shell $(PYTHON) -m delphi_utils get individual_raceeth_dir) ARCHIVE=$(shell $(PYTHON) -m delphi_utils get archive_dir) RECEIVING=$(shell $(PYTHON) -m delphi_utils get export_dir) FB_CC=$(shell $(PYTHON) -m delphi_utils get qualtrics.notify-bad-weights) @@ -26,6 +27,7 @@ MAX_WEIGHTED=ls -1 $(WEIGHTS) | grep dap | tail -1 | sed 's/_.*//;s/-//g;' ANTIJOIN:="antijoin.cids.sorted.txt" CIDS_DEST:="fb-interchange/cmu_respondent_ids" INDIVID_DEST:="fb-public-results/" +INDIVID_RACEETH_DEST:="protected-race-ethnicity-data/" RAW_DEST:="raw" # dry-run mode: generate all files, but do not post them anywhere, and disable all emails to outside parties. @@ -54,11 +56,12 @@ tidy: receiving cp params.json tidy/ mv $(RECEIVING)/*.csv tidy/$(RECEIVING) mv $(INDIVIDUAL)/*.csv* tidy/$(INDIVIDUAL) + mv $(INDIVIDUAL_RACEETH)/*.csv* tidy/$(INDIVIDUAL_RACEETH) tar -czf scratch/tidy-`date +"%Y-%m-%d-%H%M%S"`.tgz --exclude='tidy-*.tgz' tidy mv scratch/*.tgz tidy/ clean: - rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(CIDS)/*.csv + rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(INDIVIDUAL_RACEETH)/*.csv $(CIDS)/*.csv clean-archive: rm -f $(ARCHIVE)/*.Rds @@ -100,6 +103,7 @@ params.json: $(TODAY) PAT=`grep fb-survey params.json | awk 'BEGIN{FS="\""}{print $$2}' | sed 's/ /_/g;s/^/-e /'`; \ $(PYTHON) -m delphi_utils set \ debug false \ + produce_individual_raceeth true \ end_date $(YESTERDAY) \ input <(find $(QUALTRICS) -maxdepth 1 -newer $< -type f -name "*.csv" | sort | grep $${PAT} | tr '\n' ',' | sed 's_$(QUALTRICS)/__g;s/,$$//' ) \ parallel true \ @@ -140,7 +144,7 @@ run-R: $(CIDS) grep "scheduled core" tmp ; \ [ "$$?" -eq 1 ] -pipeline: scratch init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-done tidy +pipeline: scratch init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-individual-raceeth post-done tidy grep $(TODAY) params.json [ -f $(YESTERDAY) ] && rm $(YESTERDAY) || true touch $@ @@ -197,6 +201,19 @@ post-individual: $(TODAY) $(INDIVIDUAL) echo "SUCCESS: $(DRY_MESSAGE)Posted `echo $${POST} | wc -w` microresponse files" >> $(MESSAGES) touch $@ +post-individual-raceeth: $(TODAY) $(INDIVIDUAL_RACEETH) + POST=`find $(INDIVIDUAL_RACEETH) -maxdepth 1 -newer $(TODAY) -name "cvid_responses_*.csv"`; \ + [ -n "$${POST}" ]; \ + BATCH=""; \ + for f in $${POST}; do \ + (grep token $$f; [[ $$? -eq 1 ]]); \ + gzip -f $$f; \ + BATCH="$${BATCH}put $${f}.gz ${INDIVID_RACEETH_DEST}\n"; \ + done; \ + $(SFTP_POST); \ + echo "SUCCESS: $(DRY_MESSAGE)Posted `echo $${POST} | wc -w` race-ethnicity microresponse files" >> $(MESSAGES) + touch $@ + post-done: post-cids touch $(YESTERDAY).done BATCH="put $(YESTERDAY).done $(CIDS_DEST)\n"; \ diff --git a/facebook/params.json.production.template b/facebook/params.json.production.template index 2756d80dc..79dc1aaab 100644 --- a/facebook/params.json.production.template +++ b/facebook/params.json.production.template @@ -7,6 +7,8 @@ "end_date": "2020-08-28", "export_dir": "./receiving", "individual_dir": "./individual", + "individual_raceeth_dir": "./individual_raceeth", + "produce_individual_raceeth": false, "input": [ "2020-08-29.2020-08-22.2020-08-29.Survey_of_COVID-Like_Illness_-_TODEPLOY_......_-_US_Expansion.csv", "2020-08-29.2020-08-22.2020-08-29.Survey_of_COVID-Like_Illness_-_TODEPLOY-_US_Expansion_-_With_Translations.csv"