Update run script to copy a specific range of files to the receiving directory (#575)

* Add vars and one-liner to a specific range of files

* Break up long line

* Specify directory to list

* Add a bit of safety

* Remove cruft

Author: korlaxxalrok

safegraph_patterns/run-safegraph_patterns.sh

@@ -5,6 +5,11 @@
 
 #set -eo pipefail
 
+# Vars.
+# Date of first monday 8 weeks in the past. Taste to suit!
+start_day=$(date --date 'monday-56 days' +"%Y%m%d")
+today=$(date +"%Y%m%d")
+
 # Purge the receiving directory.
 echo "Purging ./receiving..."
 rm -f ./receiving/*
@@ -14,7 +19,11 @@ echo "Running the indicator..."
 env/bin/python -m delphi_safegraph_patterns
 
 # Copy the files to the ingestion directory.
+# The unwieldy one-liner does the following:
+# - Pipe a list of files into awk.
+# - awk prints the files that are inclusive of ${start_day} to ${today}.
+# - Pipe that list into xargs and copy those files to the receiving dir.
 echo "Copying files to the ingestion directory..."
-# Hack to make cp care less about missing recent files since we don't always have them.
-cp $(date +"receiving/%Y%m*") /common/covidcast/receiving/safegraph 2>/dev/null
-cp $(date --date='-1 month' +"receiving/%Y%m*") /common/covidcast/receiving/safegraph
+cd ./receiving || exit
+ls -1 *.csv | awk '$0>=from && $0<=to' from="${start_day}" to="${today}" \
+  | xargs cp -t /common/covidcast/receiving/safegraph