From 15edb9620691fefda82a1a27f1322073d68e9b47 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Mon, 29 Mar 2021 12:02:09 -0400
Subject: [PATCH 1/2] determine export days from metadata lag by default

---
 ansible/templates/google_symptoms-params-prod.json.j2 |  1 -
 google_symptoms/delphi_google_symptoms/run.py         | 10 +++++++++-
 google_symptoms/params.json.template                  |  1 -
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/ansible/templates/google_symptoms-params-prod.json.j2 b/ansible/templates/google_symptoms-params-prod.json.j2
index 70929202e..c2d7fbead 100644
--- a/ansible/templates/google_symptoms-params-prod.json.j2
+++ b/ansible/templates/google_symptoms-params-prod.json.j2
@@ -5,7 +5,6 @@
   },
   "indicator": {
     "export_start_date": "2020-02-20",
-    "num_export_days": 14,
     "bigquery_credentials": {
       "type": "{{ google_symptoms_account_type }}",
       "project_id": "{{ google_symptoms_project_id }}",
diff --git a/google_symptoms/delphi_google_symptoms/run.py b/google_symptoms/delphi_google_symptoms/run.py
index 2ef71c30b..616c0788c 100644
--- a/google_symptoms/delphi_google_symptoms/run.py
+++ b/google_symptoms/delphi_google_symptoms/run.py
@@ -7,6 +7,7 @@
 import time
 from datetime import datetime
 from itertools import product
+import covidcast
 
 import numpy as np
 from delphi_utils import (
@@ -46,7 +47,14 @@ def run_module(params):
     export_start_date = datetime.strptime(
         params["indicator"]["export_start_date"], "%Y-%m-%d")
     export_dir = params["common"]["export_dir"]
-    num_export_days = params["indicator"].get("num_export_days", "all")
+
+    if "num_export_days" in params["indicator"]:
+        num_export_days = params["indicator"]["num_export_days"]
+    else:
+        # Get number of days based on what's missing from the API.
+        metadata = covidcast.metadata()
+        gs_metadata = metadata[(metadata.data_source == "google-symptoms")]
+        num_export_days = max(gs_metadata.min_lag)
 
     logger = get_structured_logger(
         __name__, filename=params["common"].get("log_filename"),
diff --git a/google_symptoms/params.json.template b/google_symptoms/params.json.template
index 29b346b32..109773f7a 100644
--- a/google_symptoms/params.json.template
+++ b/google_symptoms/params.json.template
@@ -5,7 +5,6 @@
   },
   "indicator": {
     "export_start_date": "2020-02-20",
-    "num_export_days": 14,
     "bigquery_credentials": {}
   },
   "validation": {

From 4d455e0b55f51152c284cd4c4944ff41bb3ee231 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Wed, 31 Mar 2021 15:31:16 -0400
Subject: [PATCH 2/2] num_export_days is null by default for visibility

---
 ansible/templates/google_symptoms-params-prod.json.j2 | 1 +
 google_symptoms/delphi_google_symptoms/run.py         | 5 ++---
 google_symptoms/params.json.template                  | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ansible/templates/google_symptoms-params-prod.json.j2 b/ansible/templates/google_symptoms-params-prod.json.j2
index c2d7fbead..f91fe9606 100644
--- a/ansible/templates/google_symptoms-params-prod.json.j2
+++ b/ansible/templates/google_symptoms-params-prod.json.j2
@@ -5,6 +5,7 @@
   },
   "indicator": {
     "export_start_date": "2020-02-20",
+    "num_export_days": null,
     "bigquery_credentials": {
       "type": "{{ google_symptoms_account_type }}",
       "project_id": "{{ google_symptoms_project_id }}",
diff --git a/google_symptoms/delphi_google_symptoms/run.py b/google_symptoms/delphi_google_symptoms/run.py
index 616c0788c..569ec1821 100644
--- a/google_symptoms/delphi_google_symptoms/run.py
+++ b/google_symptoms/delphi_google_symptoms/run.py
@@ -47,10 +47,9 @@ def run_module(params):
     export_start_date = datetime.strptime(
         params["indicator"]["export_start_date"], "%Y-%m-%d")
     export_dir = params["common"]["export_dir"]
+    num_export_days = params["indicator"]["num_export_days"]
 
-    if "num_export_days" in params["indicator"]:
-        num_export_days = params["indicator"]["num_export_days"]
-    else:
+    if num_export_days is None:
         # Get number of days based on what's missing from the API.
         metadata = covidcast.metadata()
         gs_metadata = metadata[(metadata.data_source == "google-symptoms")]
diff --git a/google_symptoms/params.json.template b/google_symptoms/params.json.template
index 109773f7a..2970c29af 100644
--- a/google_symptoms/params.json.template
+++ b/google_symptoms/params.json.template
@@ -5,6 +5,7 @@
   },
   "indicator": {
     "export_start_date": "2020-02-20",
+    "num_export_days": null,
     "bigquery_credentials": {}
   },
   "validation": {