set default num days to export to all since export start date

Since BigQuery tables are currently not partitioned by date, each
query processes and bills for all rows in the table regardless of filters
applied (date and country, at the moment). To take advantage of this,
this pipeline will pull all dates from the specified start date to the
current date by default. This setting should be updated to a shorter
date window (~14 days seems reasonable) if/when the tables are converted
to "partitioned" format.

Author: nmdefries

google_symptoms/delphi_google_symptoms/pull.py

@@ -94,7 +94,7 @@ def preprocess(df, level):
     return df
 
 
-def get_date_range(export_start_date, retrieve_days_before_now):
+def get_date_range(export_start_date, num_export_days):
     """Produce date range to retrieve data for.
 
     Calculate start of date range as a static offset from the end date
@@ -105,7 +105,7 @@ def get_date_range(export_start_date, retrieve_days_before_now):
     ----------
     export_start_date: date
         first date to retrieve data for
-    retrieve_days_before_now: int
+    num_export_days: int
         number of days before end date ("now") to export
 
     Returns
@@ -115,13 +115,17 @@ def get_date_range(export_start_date, retrieve_days_before_now):
     PAD_DAYS = 7
 
     end_date = date.today()
-    # Don't fetch data before the user-set start date. Convert both
-    # dates/datetimes to date to avoid error from trying to compare
-    # different types.
-    start_date = max(
-        end_date - timedelta(days=retrieve_days_before_now),
-        export_start_date.date()
-    )
+    if num_export_days == "all":
+        # Get all dates since export_start_date.
+        start_date = export_start_date
+    else:
+        # Don't fetch data before the user-set start date. Convert both
+        # dates/datetimes to date to avoid error from trying to compare
+        # different types.
+        start_date = max(
+            end_date - timedelta(days=num_export_days),
+            export_start_date.date()
+        )
 
     retrieve_dates = [
         start_date - timedelta(days=PAD_DAYS - 1),

google_symptoms/delphi_google_symptoms/run.py

@@ -32,7 +32,7 @@ def run_module():
     export_start_date = datetime.strptime(
         params["export_start_date"], "%Y-%m-%d")
     export_dir = params["export_dir"]
-    num_export_days = params.get("num_export_days", 14)
+    num_export_days = params.get("num_export_days", "all")
 
     logger = get_structured_logger(
         __name__, filename=params.get("log_filename"),