Merge pull request #1625 from cmu-delphi/ndefries/weekly-weights-syntax

Fix up weekly weights Makefile logic

Author: krivard

.github/workflows/build-container-images.yml

@@ -2,7 +2,7 @@ name: Build indicator container images and upload to registry
 
 on:
   push:
-    branches: [ main, prod, krivard/ctis-ww-makefile ] # TODO Remove 'krivard...' branch after testing.
+    branches: [ main, prod ]
 
 jobs:
   build:

Jenkinsfile

@@ -19,7 +19,7 @@ pipeline {
     stages {
         stage('Build and Package') {
             when {
-                branch "krivard/ctis-ww-makefile"; // TODO Switch back to 'main' after testing.
+                branch "main";
             }
             steps {
                 script {
@@ -34,7 +34,7 @@ pipeline {
         }
         stage('Deploy staging') {
             when {
-                branch "krivard/ctis-ww-makefile"; // TODO Switch back to 'main' after testing.
+                branch "main";
             }
             steps {
                 script {

facebook/Makefile

@@ -70,10 +70,11 @@ tidy_%: receiving
 	rm -f $@/params.json
 	mkdir -p $@ $@/$(RECEIVING) $@/$(INDIVIDUAL) $@/$(INDIVIDUAL_RACEETH)
 	cp params.json $@/
-	mv $(RECEIVING)/*.csv $@/$(RECEIVING)
+	# Check for _any_ matching files using https://stackoverflow.com/a/6364244/14401472
+	if compgen -G "$(RECEIVING)/*.csv" > /dev/null; then mv $(RECEIVING)/*.csv $@/$(RECEIVING); fi
 	mv $(INDIVIDUAL)/*.csv* $@/$(INDIVIDUAL)
 	mv $(INDIVIDUAL_RACEETH)/*.csv* $@/$(INDIVIDUAL_RACEETH)
-	tar -czf scratch/$@-`date +"%Y-%m-%d-%H%M%S"`.tgz --exclude='tidy*-*.tgz' $@
+	tar -czf scratch/$@-`date +"%Y-%m-%d-%H%M%S"`.tgz --exclude='tidy*-*.tgz' --exclude='*.done' $@
 	mv scratch/*.tgz $@/
 
 clean:
@@ -172,31 +173,25 @@ $(WEEKLY_WEIGHTS): $(TODAY)
 	done; \
 	sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e "$${BATCH}") -P 2222 $(DELPHI_SURVEY_SFTP_USER) || exit 90; \
 	cd -; \
-# Expect weights for last week to be available if more than 3 days after the end of last week and output files for last week have not yet been created.
-	if [[ $(TODAY) -gt $(TUESDAY) && ! -f $(LAST_SUNDAY)-weekly-weights.done ]]; then \
-# Get list of start dates (Sunday) of weekly weights files.
-	  MAX_WEEKLY_WEIGHTED=`$(MAX_WEEKLY_WEIGHTED)`; \
-	  EXPECTED_MAX_WEEKLY_WEIGHTED="$(LAST_SUNDAY)"; \
-	  if [[ $$MAX_WEEKLY_WEIGHTED -lt $$EXPECTED_MAX_WEEKLY_WEIGHTED ]]; then \
-	    MSG="Weekly weights are old; Expected most recent weekly weights file to start on: $$EXPECTED_MAX_WEEKLY_WEIGHTED; Actual most recent file starts on: $$MAX_WEEKLY_WEIGHTED"; \
-	    echo "WARNING: $${MSG}" >> $(MESSAGES); \
-	  fi; \
-	fi
-# If any new files are downloaded, run weekly weights pipeline for last week.
-	if [[ $(wc -w <<< $${NEW}) -gt 0 ]]; then \
-# The earliest new weekly weights file should be for last week.
-	  MIN_NEW_WEEKLY_WEIGHTED=`grep map <<< $${NEW} | head -1 | sed 's/_.*//;s/-//g;'`; \
-	  EXPECTED_NEW_WEEKLY_WEIGHTED="$(LAST_SUNDAY)"; \
-	  if [[ $$EXPECTED_NEW_WEEKLY_WEIGHTED -ne $$MIN_NEW_WEEKLY_WEIGHTED ]]; then \
-		   MSG="Expected new weekly weights file to start on: $$EXPECTED_NEW_WEEKLY_WEIGHTED; Actual new file starts on: $$MIN_NEW_WEEKLY_WEIGHTED"; \
-		   echo "WARNING: $${MSG}" >> $(MESSAGES); \
-	  fi; \
-# Save existing params file from normal daily run.
+	touch -d $(YESTERDAY) $(WEEKLY_WEIGHTS); \
+	EXPECTED_WEEKLY_WEIGHTED=`date --date='$(LAST_SUNDAY)' +'%Y%m%d'`; \
+	MIN_NEW_WEEKLY_WEIGHTED=`grep map <<< $${NEW} | head -1 | sed 's/_.*//;s/-//g;'`; \
+	if [[ `wc -w <<< $${NEW}` -gt 0 ]] && [[ $$MIN_NEW_WEEKLY_WEIGHTED -ne $$EXPECTED_WEEKLY_WEIGHTED ]]; then \
+	  MSG="Expected new weekly weights files to start on: $$EXPECTED_WEEKLY_WEIGHTED; Actual new files starts on: $$MIN_NEW_WEEKLY_WEIGHTED"; \
+	  echo "WARNING: $${MSG}" >> $(MESSAGES); \
+	fi; \
+	MAX_WEEKLY_WEIGHTED=`$(MAX_WEEKLY_WEIGHTED)`; \
+	if [[ `date --date='$(TODAY)' +'%Y%m%d'` -gt `date --date='$(TUESDAY)' +'%Y%m%d'` ]] && [[ $$MAX_WEEKLY_WEIGHTED -lt $$EXPECTED_WEEKLY_WEIGHTED ]]; then \
+	  MSG="Weekly weights are old; Expected most recent weekly weights file to start on: $$EXPECTED_WEEKLY_WEIGHTED; Actual most recent file starts on: $$MAX_WEEKLY_WEIGHTED"; \
+	  echo "WARNING: $${MSG}" >> $(MESSAGES); \
+	fi; \
+	if [[ ! -f tidy_weekly/$(LAST_SUNDAY)-weekly-weights.done ]] && [[ $$MAX_WEEKLY_WEIGHTED -eq $$EXPECTED_WEEKLY_WEIGHTED ]]; then \
 	  if [ -f params.json ]; then cp params.json params.daily.json; fi; \
 	  $(MAKE) weekly-weights-pipeline; \
-	  rm -f params.json; \
-# Restore params file from normal daily run.
-	  if [ -f params.daily.json ]; then mv params.daily.json params.json; fi; \
+	  if [ -f params.daily.json ]; then \
+		cp params.daily.json params.json; \
+		rm -f params.daily.json; \
+	  fi; \
 	fi
 
 dev: delphiFacebook_1.0.tar.gz
@@ -222,7 +217,7 @@ pipeline: scratch init-qualtrics params.json $(WEIGHTS) run-R post-cids post-exp
 weekly-weights-pipeline: scratch init-qualtrics params.weekly-weights.json run-R post-individual post-individual-raceeth tidy_weekly
 	touch $@
 	echo "SUCCESS: $(DRY_MESSAGE)completed weekly weights pipeline" >> $(MESSAGES)
-	touch $(LAST_SUNDAY)-weekly-weights.done
+	touch tidy_weekly/$(LAST_SUNDAY)-weekly-weights.done
 
 coverage:
 	Rscript -e 'covr::package_coverage("delphiFacebook")'