Skip to content

Support upload of raceeth microdata in Makefile #1211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 11, 2021
Merged

Support upload of raceeth microdata in Makefile #1211

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4318312
support upload of raceeth microdata in Makefil
nmdefries Aug 18, 2021
6fd1bee
Merge branch 'main' into produce-microdata-raceeth
nmdefries Aug 18, 2021
c7f63d4
add new keys to params prod template; turn on raceeth microdata in make
nmdefries Aug 18, 2021
1035362
Merge branch 'main' into produce-microdata-raceeth
nmdefries Oct 7, 2021
9d3a8b5
Merge branch 'main' into produce-microdata-raceeth
nmdefries Oct 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions facebook/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ QUALTRICS=$(shell $(PYTHON) -m delphi_utils get input_dir)
WEIGHTS=$(shell $(PYTHON) -m delphi_utils get weights_in_dir)
CIDS=$(shell $(PYTHON) -m delphi_utils get weights_out_dir)
INDIVIDUAL=$(shell $(PYTHON) -m delphi_utils get individual_dir)
INDIVIDUAL_RACEETH=$(shell $(PYTHON) -m delphi_utils get individual_raceeth_dir)
ARCHIVE=$(shell $(PYTHON) -m delphi_utils get archive_dir)
RECEIVING=$(shell $(PYTHON) -m delphi_utils get export_dir)
FB_CC=$(shell $(PYTHON) -m delphi_utils get qualtrics.notify-bad-weights)
Expand All @@ -33,6 +34,7 @@ MAX_WEIGHTED=ls -1 $(WEIGHTS) | grep dap | tail -1 | sed 's/_.*//;s/-//g;'
ANTIJOIN:="antijoin.cids.sorted.txt"
CIDS_DEST:="fb-interchange/cmu_respondent_ids"
INDIVID_DEST:="fb-public-results/"
INDIVID_RACEETH_DEST:="protected-race-ethnicity-data/"
RAW_DEST:="raw"

default:
Expand All @@ -44,10 +46,11 @@ tidy: receiving
cp params.json tidy/
mv $(RECEIVING)/*.csv tidy/$(RECEIVING)
mv $(INDIVIDUAL)/*.csv* tidy/$(INDIVIDUAL)
mv $(INDIVIDUAL_RACEETH)/*.csv* tidy/$(INDIVIDUAL_RACEETH)
tar -czf tidy-`date +"%Y-%m-%d-%H%M%S"`.tgz tidy

clean:
rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(CIDS)/*.csv
rm -f $(RECEIVING)/*.csv $(INDIVIDUAL)/*.csv $(INDIVIDUAL_RACEETH)/*.csv $(CIDS)/*.csv

clean-archive:
rm -f $(ARCHIVE)/*.Rds
Expand Down Expand Up @@ -89,6 +92,7 @@ params.json: $(TODAY)
PAT=`grep fb-survey params.json | awk 'BEGIN{FS="\""}{print $$2}' | sed 's/ /_/g;s/^/-e /'`; \
$(PYTHON) -m delphi_utils set \
debug false \
produce_individual_raceeth true \
end_date $(YESTERDAY) \
input <(find $(QUALTRICS) -maxdepth 1 -newer $< -type f -name "*.csv" | sort | grep $${PAT} | tr '\n' ',' | sed 's_$(QUALTRICS)/__g;s/,$$//' ) \
parallel true \
Expand Down Expand Up @@ -129,7 +133,7 @@ run-R: $(CIDS)
grep "scheduled core" tmp ; \
[ "$$?" -eq 1 ]

pipeline: init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-done tidy
pipeline: init-qualtrics params.json $(WEIGHTS) run-R post-cids post-individual post-individual-raceeth post-done tidy
grep $(TODAY) params.json
[ -f $(YESTERDAY) ] && rm $(YESTERDAY) || true
touch $@
Expand Down Expand Up @@ -185,6 +189,19 @@ post-individual: $(TODAY) $(INDIVIDUAL)
echo "SUCCESS: $(DRY_MESSAGE)Posted `echo $${POST} | wc -w` microresponse files" >> $(MESSAGES)
touch $@

post-individual-raceeth: $(TODAY) $(INDIVIDUAL_RACEETH)
POST=`find $(INDIVIDUAL_RACEETH) -maxdepth 1 -newer $(TODAY) -name "cvid_responses_*.csv"`; \
[ -n "$${POST}" ]; \
BATCH=""; \
for f in $${POST}; do \
(grep token $$f; [[ $$? -eq 1 ]]); \
gzip -f $$f; \
BATCH="$${BATCH}put $${f}.gz ${INDIVID_RACEETH_DEST}\n"; \
done; \
$(SFTP_POST); \
echo "SUCCESS: $(DRY_MESSAGE)Posted `echo $${POST} | wc -w` race-ethnicity microresponse files" >> $(MESSAGES)
touch $@

post-done: post-cids
touch $(YESTERDAY).done
BATCH="put $(YESTERDAY).done $(CIDS_DEST)\n"; \
Expand Down
2 changes: 2 additions & 0 deletions facebook/params.json.production.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
"end_date": "2020-08-28",
"export_dir": "./receiving",
"individual_dir": "./individual",
"individual_raceeth_dir": "./individual_raceeth",
"produce_individual_raceeth": false,
"input": [
"2020-08-29.2020-08-22.2020-08-29.Survey_of_COVID-Like_Illness_-_TODEPLOY_......_-_US_Expansion.csv",
"2020-08-29.2020-08-22.2020-08-29.Survey_of_COVID-Like_Illness_-_TODEPLOY-_US_Expansion_-_With_Translations.csv"
Expand Down