diff --git a/facebook/Dockerfile b/facebook/Dockerfile index d78599349..47969ca58 100644 --- a/facebook/Dockerfile +++ b/facebook/Dockerfile @@ -27,6 +27,7 @@ ADD ./monthly-files.R /facebook/monthly-files.R ADD ./contingency_tables.R /facebook/contingency_tables.R ADD ./contingency-combine.R /facebook/contingency-combine.R ADD ./ssmtp.conf /etc/ssmtp/ssmtp.conf +ADD ./GITREF facebook/GITREF WORKDIR /facebook/ RUN make lib RUN make install diff --git a/facebook/GITREF b/facebook/GITREF new file mode 100644 index 000000000..e69de29bb diff --git a/facebook/Makefile b/facebook/Makefile index 3ed8824d8..1482ff282 100644 --- a/facebook/Makefile +++ b/facebook/Makefile @@ -31,11 +31,9 @@ RAW_DEST:="raw" # dry-run mode: generate all files, but do not post them anywhere, and disable all emails to outside parties. DRY:=yes ifeq ($(DRY),yes) - EMAIL_SEND:=echo -e "Would send mail: echo -e \"Subject: $${SUBJECT}\n\n$${MSG}\" | sendmail $(DELPHI_SURVEY_EMAIL_USER)" SFTP_POST:=echo -e "Would run: sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e \"\$${BATCH}\") -P 2222 $(DELPHI_SURVEY_SFTP_USER)\n$${BATCH}" DRY_MESSAGE:="[DRY-RUN] " else - EMAIL_SEND:=echo -e "Subject: $${SUBJECT}\n\n$${MSG}" | sendmail $(DELPHI_SURVEY_EMAIL_USER) SFTP_POST:=sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e "$${BATCH}") -P 2222 $(DELPHI_SURVEY_SFTP_USER) endif @@ -121,10 +119,8 @@ $(WEIGHTS): $(TODAY) MAX_WEIGHTED=`$(MAX_WEIGHTED)`; \ EXPECTED_MAX_WEIGHTED=`date --date='$(TODAY) -3 day' +'%Y%m%d'`; \ if [[ $$EXPECTED_MAX_WEIGHTED -gt $$MAX_WEIGHTED ]]; then \ - MSG="Expected most recent file: $$EXPECTED_MAX_WEIGHTED\nActual most recent file: $$MAX_WEIGHTED"; \ - echo "WARNING: $${MSG}" | tr "\n" ";" >> $(MESSAGES); \ - SUBJECT="[fb-cmu-cvid] Weights are stale"; \ - $(EMAIL_SEND) ;\ + MSG="Expected most recent file: $$EXPECTED_MAX_WEIGHTED; Actual most recent file: $$MAX_WEIGHTED"; \ + echo "WARNING: $${MSG}" >> $(MESSAGES); \ fi dev: delphiFacebook_1.0.tar.gz diff --git a/facebook/micro/monthly-archive.sh b/facebook/micro/monthly-archive.sh new file mode 100644 index 000000000..15a50890e --- /dev/null +++ b/facebook/micro/monthly-archive.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +rm -f *.gz +if [ -z $1 ]; then + MONTH=`date --date "last month" +"%Y_%m"` +else + MONTH=$1 +fi +echo ${MONTH} +R_MONTH=${MONTH#*_}; R_MONTH=${R_MONTH#0} +BATCH="cd fb-public-results\nls -1 cvid_responses_${MONTH}*.gz" +sftp -b <(echo -e "${BATCH}") -P 2222 fb-automation@ftp.delphi.cmu.edu 2>/dev/null | \ + grep "^cvid" | \ + awk -F_ 'BEGIN{print "cd fb-public-results"} {key=$3 $4 $5; if (key!=last && last!="") {print record} last=key; record=$0} END{print record}' | \ + sed '/^cvid/ s/^/get /' >fetch.sftp +sftp -b fetch.sftp -P 2222 fb-automation@ftp.delphi.cmu.edu +OUT=${MONTH/_/-} +Rscript ../monthly-files.R ${MONTH%_*} ${R_MONTH} . >${OUT}.csv +gzip ${OUT}.csv +sftp -b <(echo -e "cd fb-public-results\nput ${OUT}.csv.gz") -P 2222 fb-automation@ftp.delphi.cmu.edu