From 15edb9620691fefda82a1a27f1322073d68e9b47 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Mon, 29 Mar 2021 12:02:09 -0400
Subject: [PATCH 1/6] determine export days from metadata lag by default

---
 ansible/templates/google_symptoms-params-prod.json.j2 |  1 -
 google_symptoms/delphi_google_symptoms/run.py         | 10 +++++++++-
 google_symptoms/params.json.template                  |  1 -
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/ansible/templates/google_symptoms-params-prod.json.j2 b/ansible/templates/google_symptoms-params-prod.json.j2
index 70929202e..c2d7fbead 100644
--- a/ansible/templates/google_symptoms-params-prod.json.j2
+++ b/ansible/templates/google_symptoms-params-prod.json.j2
@@ -5,7 +5,6 @@
   },
   "indicator": {
     "export_start_date": "2020-02-20",
-    "num_export_days": 14,
     "bigquery_credentials": {
       "type": "{{ google_symptoms_account_type }}",
       "project_id": "{{ google_symptoms_project_id }}",
diff --git a/google_symptoms/delphi_google_symptoms/run.py b/google_symptoms/delphi_google_symptoms/run.py
index 2ef71c30b..616c0788c 100644
--- a/google_symptoms/delphi_google_symptoms/run.py
+++ b/google_symptoms/delphi_google_symptoms/run.py
@@ -7,6 +7,7 @@
 import time
 from datetime import datetime
 from itertools import product
+import covidcast
 
 import numpy as np
 from delphi_utils import (
@@ -46,7 +47,14 @@ def run_module(params):
     export_start_date = datetime.strptime(
         params["indicator"]["export_start_date"], "%Y-%m-%d")
     export_dir = params["common"]["export_dir"]
-    num_export_days = params["indicator"].get("num_export_days", "all")
+
+    if "num_export_days" in params["indicator"]:
+        num_export_days = params["indicator"]["num_export_days"]
+    else:
+        # Get number of days based on what's missing from the API.
+        metadata = covidcast.metadata()
+        gs_metadata = metadata[(metadata.data_source == "google-symptoms")]
+        num_export_days = max(gs_metadata.min_lag)
 
     logger = get_structured_logger(
         __name__, filename=params["common"].get("log_filename"),
diff --git a/google_symptoms/params.json.template b/google_symptoms/params.json.template
index 29b346b32..109773f7a 100644
--- a/google_symptoms/params.json.template
+++ b/google_symptoms/params.json.template
@@ -5,7 +5,6 @@
   },
   "indicator": {
     "export_start_date": "2020-02-20",
-    "num_export_days": 14,
     "bigquery_credentials": {}
   },
   "validation": {

From e9fa8c2e2d35ca4266fd9bbdbb63d111fed3e3df Mon Sep 17 00:00:00 2001
From: Ben Smith <benjamin.y.smith@gmail.com>
Date: Wed, 31 Mar 2021 15:28:30 -0400
Subject: [PATCH 2/6] Don't set file output in root logger

---
 _delphi_utils_python/delphi_utils/logger.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/_delphi_utils_python/delphi_utils/logger.py b/_delphi_utils_python/delphi_utils/logger.py
index 62f2ff460..35560114e 100644
--- a/_delphi_utils_python/delphi_utils/logger.py
+++ b/_delphi_utils_python/delphi_utils/logger.py
@@ -40,16 +40,12 @@ def get_structured_logger(name=__name__,
     is a good choice.
     filename: An (optional) file to write log output.
     """
-    # Configure the underlying logging configuration
-    handlers = [logging.StreamHandler()]
-    if filename:
-        handlers.append(logging.FileHandler(filename))
-
+    # Configure the basic underlying logging configuration
     logging.basicConfig(
         format="%(message)s",
         level=logging.INFO,
-        handlers=handlers
-        )
+        handlers=[logging.StreamHandler()]
+    )    
 
     # Configure structlog. This uses many of the standard suggestions from
     # the structlog documentation.
@@ -84,7 +80,12 @@ def get_structured_logger(name=__name__,
         cache_logger_on_first_use=True,
     )
 
-    logger = structlog.get_logger(name)
+    # Create the underlying python logger and wrap it with structlog
+    system_logger = logging.getLogger(name)
+    if filename:
+        system_logger.addHandler(logging.FileHandler(filename))
+    system_logger.setLevel(logging.INFO)
+    logger = structlog.wrap_logger(system_logger)
 
     if log_exceptions:
         handle_exceptions(logger)

From 84cb1c820799484cc9fe819c895fb9a7afb96a7b Mon Sep 17 00:00:00 2001
From: Mike O'Brien <mikeob@google.com>
Date: Wed, 31 Mar 2021 15:28:56 -0400
Subject: [PATCH 3/6] change logging schemefor validation

---
 _delphi_utils_python/delphi_utils/runner.py         |  2 ++
 .../delphi_utils/validator/report.py                | 13 +++++++------
 _delphi_utils_python/delphi_utils/validator/run.py  | 10 +++++++---
 _delphi_utils_python/tests/test_runner.py           |  1 +
 _delphi_utils_python/tests/validator/test_report.py |  6 +++---
 5 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/_delphi_utils_python/delphi_utils/runner.py b/_delphi_utils_python/delphi_utils/runner.py
index 8547b2e5f..66f0b9379 100644
--- a/_delphi_utils_python/delphi_utils/runner.py
+++ b/_delphi_utils_python/delphi_utils/runner.py
@@ -3,6 +3,7 @@
 import importlib
 from typing import Any, Callable, Dict, Optional
 from .archive import ArchiveDiffer, archiver_from_params
+from .logger import get_structured_logger
 from .utils import read_params
 from .validator.validate import Validator
 from .validator.run import validator_from_params
@@ -40,6 +41,7 @@ def run_indicator_pipeline(indicator_fn:  Callable[[Params], None],
     archiver = archiver_fn(params)
     if validator:
         validation_report = validator.validate()
+        validation_report.log(get_structured_logger(params["common"].get("log_filename", None)))
     if archiver and (not validator or validation_report.success()):
         archiver.archive()
 
diff --git a/_delphi_utils_python/delphi_utils/validator/report.py b/_delphi_utils_python/delphi_utils/validator/report.py
index 7c34a3fac..0bdb0948c 100644
--- a/_delphi_utils_python/delphi_utils/validator/report.py
+++ b/_delphi_utils_python/delphi_utils/validator/report.py
@@ -4,12 +4,10 @@
 from ..logger import get_structured_logger
 from .errors import ValidationFailure
 
-logger = get_structured_logger(__name__)
-
 class ValidationReport:
     """Class for reporting the results of validation."""
 
-    def __init__(self, errors_to_suppress: List[ValidationFailure]):
+    def __init__(self, errors_to_suppress: List[ValidationFailure], log_filename=None):
         """Initialize a ValidationReport.
 
         Parameters
@@ -83,14 +81,17 @@ def summary(self):
         out_str += f"{len(self.raised_warnings)} warnings\n"
         return out_str
 
-    def log(self):
+    def log(self, logger=None):
         """Log errors and warnings."""
+        if logger is None:
+            logger = get_structured_logger(__name__)
+
         for error in self.unsuppressed_errors:
             logger.critical(str(error))
         for warning in self.raised_warnings:
             logger.warning(str(warning))
 
-    def print_and_exit(self, die_on_failures=True):
+    def print_and_exit(self, logger=None, die_on_failures=True):
         """Print results and exit.
 
         Arguments
@@ -99,7 +100,7 @@ def print_and_exit(self, die_on_failures=True):
             Whether to return non-zero status if any failures were encountered.
         """
         print(self.summary())
-        self.log()
+        self.log(logger)
         if self.success():
             sys.exit(0)
         elif die_on_failures:
diff --git a/_delphi_utils_python/delphi_utils/validator/run.py b/_delphi_utils_python/delphi_utils/validator/run.py
index e39fe2d44..46aca56b2 100644
--- a/_delphi_utils_python/delphi_utils/validator/run.py
+++ b/_delphi_utils_python/delphi_utils/validator/run.py
@@ -5,7 +5,7 @@
 when the module is run with `python -m delphi_utils.validator`.
 """
 import argparse as ap
-from .. import read_params
+from .. import read_params, get_structured_logger
 from .validate import Validator
 
 
@@ -15,8 +15,12 @@ def run_module():
     parser.add_argument("--dry_run", action="store_true", help="When provided, return zero exit"
                         " status irrespective of the number of failures")
     args = parser.parse_args()
-    validator = Validator(read_params())
-    validator.validate().print_and_exit(not args.dry_run)
+    params = read_params()
+    validator = Validator(params)
+    validator.validate().print_and_exit(
+        get_structured_logger(__name__,
+                              params["common"].get("log_filename", None),
+                              not args.dry_run))
 
 
 def validator_from_params(params):
diff --git a/_delphi_utils_python/tests/test_runner.py b/_delphi_utils_python/tests/test_runner.py
index 0f0bcf210..cbf7e620a 100644
--- a/_delphi_utils_python/tests/test_runner.py
+++ b/_delphi_utils_python/tests/test_runner.py
@@ -35,6 +35,7 @@ class TestRunIndicator:
     """Fixture for running indicators."""
     # arbitrary params to pass to function generators
     PARAMS = {
+        "common": {},
         "indicator": {"a": 1},
         "validation": {"b": 2},
         "archive": {"c": 3}
diff --git a/_delphi_utils_python/tests/validator/test_report.py b/_delphi_utils_python/tests/validator/test_report.py
index dc04fbd28..81f835695 100644
--- a/_delphi_utils_python/tests/validator/test_report.py
+++ b/_delphi_utils_python/tests/validator/test_report.py
@@ -44,9 +44,9 @@ def test_str(self):
         assert report.summary() ==\
             "3 checks run\n1 checks failed\n1 checks suppressed\n2 warnings\n"
 
-    @mock.patch("delphi_utils.validator.report.logger")
-    def test_log(self, mock_logger):
+    def test_log(self):
         """Test that the logs contain all failures and warnings."""
+        mock_logger = mock.Mock()
         report = ValidationReport([self.ERROR_1])
         report.increment_total_checks()
         report.increment_total_checks()
@@ -56,7 +56,7 @@ def test_log(self, mock_logger):
         report.add_raised_error(self.ERROR_1)
         report.add_raised_error(self.ERROR_2)
 
-        report.log()
+        report.log(mock_logger)
         mock_logger.critical.assert_called_once_with(
             "bad failed for sig2 at resolution county on 2020-11-07: msg 2")
         mock_logger.warning.assert_has_calls([mock.call("wrong import"), mock.call("right import")])

From 4d455e0b55f51152c284cd4c4944ff41bb3ee231 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Wed, 31 Mar 2021 15:31:16 -0400
Subject: [PATCH 4/6] num_export_days is null by default for visibility

---
 ansible/templates/google_symptoms-params-prod.json.j2 | 1 +
 google_symptoms/delphi_google_symptoms/run.py         | 5 ++---
 google_symptoms/params.json.template                  | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ansible/templates/google_symptoms-params-prod.json.j2 b/ansible/templates/google_symptoms-params-prod.json.j2
index c2d7fbead..f91fe9606 100644
--- a/ansible/templates/google_symptoms-params-prod.json.j2
+++ b/ansible/templates/google_symptoms-params-prod.json.j2
@@ -5,6 +5,7 @@
   },
   "indicator": {
     "export_start_date": "2020-02-20",
+    "num_export_days": null,
     "bigquery_credentials": {
       "type": "{{ google_symptoms_account_type }}",
       "project_id": "{{ google_symptoms_project_id }}",
diff --git a/google_symptoms/delphi_google_symptoms/run.py b/google_symptoms/delphi_google_symptoms/run.py
index 616c0788c..569ec1821 100644
--- a/google_symptoms/delphi_google_symptoms/run.py
+++ b/google_symptoms/delphi_google_symptoms/run.py
@@ -47,10 +47,9 @@ def run_module(params):
     export_start_date = datetime.strptime(
         params["indicator"]["export_start_date"], "%Y-%m-%d")
     export_dir = params["common"]["export_dir"]
+    num_export_days = params["indicator"]["num_export_days"]
 
-    if "num_export_days" in params["indicator"]:
-        num_export_days = params["indicator"]["num_export_days"]
-    else:
+    if num_export_days is None:
         # Get number of days based on what's missing from the API.
         metadata = covidcast.metadata()
         gs_metadata = metadata[(metadata.data_source == "google-symptoms")]
diff --git a/google_symptoms/params.json.template b/google_symptoms/params.json.template
index 109773f7a..2970c29af 100644
--- a/google_symptoms/params.json.template
+++ b/google_symptoms/params.json.template
@@ -5,6 +5,7 @@
   },
   "indicator": {
     "export_start_date": "2020-02-20",
+    "num_export_days": null,
     "bigquery_credentials": {}
   },
   "validation": {

From 3dcefe843d1ea5a20b09fb67fb86ed558cb0cc44 Mon Sep 17 00:00:00 2001
From: Ben Smith <benjamin.y.smith@gmail.com>
Date: Wed, 31 Mar 2021 15:34:35 -0400
Subject: [PATCH 5/6] Whitespace

---
 _delphi_utils_python/delphi_utils/logger.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_delphi_utils_python/delphi_utils/logger.py b/_delphi_utils_python/delphi_utils/logger.py
index 35560114e..0b7a58032 100644
--- a/_delphi_utils_python/delphi_utils/logger.py
+++ b/_delphi_utils_python/delphi_utils/logger.py
@@ -45,7 +45,7 @@ def get_structured_logger(name=__name__,
         format="%(message)s",
         level=logging.INFO,
         handlers=[logging.StreamHandler()]
-    )    
+    )
 
     # Configure structlog. This uses many of the standard suggestions from
     # the structlog documentation.

From 225227379f9dd9d4f39273d2027ac934b1e859ba Mon Sep 17 00:00:00 2001
From: Mike O'Brien <mikeob@google.com>
Date: Wed, 31 Mar 2021 16:09:20 -0400
Subject: [PATCH 6/6] lint fix"

---
 _delphi_utils_python/delphi_utils/validator/report.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_delphi_utils_python/delphi_utils/validator/report.py b/_delphi_utils_python/delphi_utils/validator/report.py
index 0bdb0948c..b6d32c2cb 100644
--- a/_delphi_utils_python/delphi_utils/validator/report.py
+++ b/_delphi_utils_python/delphi_utils/validator/report.py
@@ -7,7 +7,7 @@
 class ValidationReport:
     """Class for reporting the results of validation."""
 
-    def __init__(self, errors_to_suppress: List[ValidationFailure], log_filename=None):
+    def __init__(self, errors_to_suppress: List[ValidationFailure]):
         """Initialize a ValidationReport.
 
         Parameters