Skip to content

Commit 02016fb

Browse files
authored
Merge pull request #1625 from cmu-delphi/ndefries/weekly-weights-syntax
Fix up weekly weights Makefile logic
2 parents e775f42 + 1be5421 commit 02016fb

File tree

5 files changed

+444
-32
lines changed

5 files changed

+444
-32
lines changed

.github/workflows/build-container-images.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Build indicator container images and upload to registry
22

33
on:
44
push:
5-
branches: [ main, prod, krivard/ctis-ww-makefile ] # TODO Remove 'krivard...' branch after testing.
5+
branches: [ main, prod ]
66

77
jobs:
88
build:

Jenkinsfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ pipeline {
1919
stages {
2020
stage('Build and Package') {
2121
when {
22-
branch "krivard/ctis-ww-makefile"; // TODO Switch back to 'main' after testing.
22+
branch "main";
2323
}
2424
steps {
2525
script {
@@ -34,7 +34,7 @@ pipeline {
3434
}
3535
stage('Deploy staging') {
3636
when {
37-
branch "krivard/ctis-ww-makefile"; // TODO Switch back to 'main' after testing.
37+
branch "main";
3838
}
3939
steps {
4040
script {

facebook/Makefile

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,11 @@ tidy_%: receiving
7070
rm -f $@/params.json
7171
mkdir -p $@ $@/$(RECEIVING) $@/$(INDIVIDUAL) $@/$(INDIVIDUAL_RACEETH)
7272
cp params.json $@/
73-
mv $(RECEIVING)/*.csv $@/$(RECEIVING)
73+
# Check for _any_ matching files using https://stackoverflow.com/a/6364244/14401472
74+
if compgen -G "$(RECEIVING)/*.csv" > /dev/null; then mv $(RECEIVING)/*.csv $@/$(RECEIVING); fi
7475
mv $(INDIVIDUAL)/*.csv* $@/$(INDIVIDUAL)
7576
mv $(INDIVIDUAL_RACEETH)/*.csv* $@/$(INDIVIDUAL_RACEETH)
76-
tar -czf scratch/$@-`date +"%Y-%m-%d-%H%M%S"`.tgz --exclude='tidy*-*.tgz' $@
77+
tar -czf scratch/$@-`date +"%Y-%m-%d-%H%M%S"`.tgz --exclude='tidy*-*.tgz' --exclude='*.done' $@
7778
mv scratch/*.tgz $@/
7879

7980
clean:
@@ -172,31 +173,25 @@ $(WEEKLY_WEIGHTS): $(TODAY)
172173
done; \
173174
sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e "$${BATCH}") -P 2222 $(DELPHI_SURVEY_SFTP_USER) || exit 90; \
174175
cd -; \
175-
# Expect weights for last week to be available if more than 3 days after the end of last week and output files for last week have not yet been created.
176-
if [[ $(TODAY) -gt $(TUESDAY) && ! -f $(LAST_SUNDAY)-weekly-weights.done ]]; then \
177-
# Get list of start dates (Sunday) of weekly weights files.
178-
MAX_WEEKLY_WEIGHTED=`$(MAX_WEEKLY_WEIGHTED)`; \
179-
EXPECTED_MAX_WEEKLY_WEIGHTED="$(LAST_SUNDAY)"; \
180-
if [[ $$MAX_WEEKLY_WEIGHTED -lt $$EXPECTED_MAX_WEEKLY_WEIGHTED ]]; then \
181-
MSG="Weekly weights are old; Expected most recent weekly weights file to start on: $$EXPECTED_MAX_WEEKLY_WEIGHTED; Actual most recent file starts on: $$MAX_WEEKLY_WEIGHTED"; \
182-
echo "WARNING: $${MSG}" >> $(MESSAGES); \
183-
fi; \
184-
fi
185-
# If any new files are downloaded, run weekly weights pipeline for last week.
186-
if [[ $(wc -w <<< $${NEW}) -gt 0 ]]; then \
187-
# The earliest new weekly weights file should be for last week.
188-
MIN_NEW_WEEKLY_WEIGHTED=`grep map <<< $${NEW} | head -1 | sed 's/_.*//;s/-//g;'`; \
189-
EXPECTED_NEW_WEEKLY_WEIGHTED="$(LAST_SUNDAY)"; \
190-
if [[ $$EXPECTED_NEW_WEEKLY_WEIGHTED -ne $$MIN_NEW_WEEKLY_WEIGHTED ]]; then \
191-
MSG="Expected new weekly weights file to start on: $$EXPECTED_NEW_WEEKLY_WEIGHTED; Actual new file starts on: $$MIN_NEW_WEEKLY_WEIGHTED"; \
192-
echo "WARNING: $${MSG}" >> $(MESSAGES); \
193-
fi; \
194-
# Save existing params file from normal daily run.
176+
touch -d $(YESTERDAY) $(WEEKLY_WEIGHTS); \
177+
EXPECTED_WEEKLY_WEIGHTED=`date --date='$(LAST_SUNDAY)' +'%Y%m%d'`; \
178+
MIN_NEW_WEEKLY_WEIGHTED=`grep map <<< $${NEW} | head -1 | sed 's/_.*//;s/-//g;'`; \
179+
if [[ `wc -w <<< $${NEW}` -gt 0 ]] && [[ $$MIN_NEW_WEEKLY_WEIGHTED -ne $$EXPECTED_WEEKLY_WEIGHTED ]]; then \
180+
MSG="Expected new weekly weights files to start on: $$EXPECTED_WEEKLY_WEIGHTED; Actual new files starts on: $$MIN_NEW_WEEKLY_WEIGHTED"; \
181+
echo "WARNING: $${MSG}" >> $(MESSAGES); \
182+
fi; \
183+
MAX_WEEKLY_WEIGHTED=`$(MAX_WEEKLY_WEIGHTED)`; \
184+
if [[ `date --date='$(TODAY)' +'%Y%m%d'` -gt `date --date='$(TUESDAY)' +'%Y%m%d'` ]] && [[ $$MAX_WEEKLY_WEIGHTED -lt $$EXPECTED_WEEKLY_WEIGHTED ]]; then \
185+
MSG="Weekly weights are old; Expected most recent weekly weights file to start on: $$EXPECTED_WEEKLY_WEIGHTED; Actual most recent file starts on: $$MAX_WEEKLY_WEIGHTED"; \
186+
echo "WARNING: $${MSG}" >> $(MESSAGES); \
187+
fi; \
188+
if [[ ! -f tidy_weekly/$(LAST_SUNDAY)-weekly-weights.done ]] && [[ $$MAX_WEEKLY_WEIGHTED -eq $$EXPECTED_WEEKLY_WEIGHTED ]]; then \
195189
if [ -f params.json ]; then cp params.json params.daily.json; fi; \
196190
$(MAKE) weekly-weights-pipeline; \
197-
rm -f params.json; \
198-
# Restore params file from normal daily run.
199-
if [ -f params.daily.json ]; then mv params.daily.json params.json; fi; \
191+
if [ -f params.daily.json ]; then \
192+
cp params.daily.json params.json; \
193+
rm -f params.daily.json; \
194+
fi; \
200195
fi
201196

202197
dev: delphiFacebook_1.0.tar.gz
@@ -222,7 +217,7 @@ pipeline: scratch init-qualtrics params.json $(WEIGHTS) run-R post-cids post-exp
222217
weekly-weights-pipeline: scratch init-qualtrics params.weekly-weights.json run-R post-individual post-individual-raceeth tidy_weekly
223218
touch $@
224219
echo "SUCCESS: $(DRY_MESSAGE)completed weekly weights pipeline" >> $(MESSAGES)
225-
touch $(LAST_SUNDAY)-weekly-weights.done
220+
touch tidy_weekly/$(LAST_SUNDAY)-weekly-weights.done
226221

227222
coverage:
228223
Rscript -e 'covr::package_coverage("delphiFacebook")'

0 commit comments

Comments
 (0)