Merge pull request #806 from sgsmob/google_symptoms

Refactor parameters for Google symptoms

Author: krivard

ansible/templates/google_symptoms-params-prod.json.j2

@@ -1,8 +1,12 @@
 {
-  "export_start_date": "2020-02-20",
-  "static_file_dir": "./static",
-  "export_dir": "/common/covidcast/receiving/google-symptoms",
-  "cache_dir": "./cache",
-  "base_url": "https://raw.githubusercontent.com/google-research/open-covid-19-data/master/data/exports/search_trends_symptoms_dataset/United%20States%20of%20America{sub_url}2020_US_{state}daily_symptoms_dataset.csv",
-  "log_filename": "/var/log/indicators/google_symptoms.log"
+  "common": {
+    "export_dir": "/common/covidcast/receiving/google-symptoms",
+    "log_filename": "/var/log/indicators/google_symptoms.log"
+  },
+  "indicator": {
+    "export_start_date": "2020-02-20",
+    "num_export_days": 14,
+    "path_to_bigquery_credentials": "credentials.json",
+    "static_file_dir": "./static"
+  }
 }

google_symptoms/delphi_google_symptoms/run.py

@@ -30,16 +30,16 @@ def run_module():
 
     params = read_params()
     export_start_date = datetime.strptime(
-        params["export_start_date"], "%Y-%m-%d")
-    export_dir = params["export_dir"]
-    num_export_days = params.get("num_export_days", "all")
+        params["indicator"]["export_start_date"], "%Y-%m-%d")
+    export_dir = params["common"]["export_dir"]
+    num_export_days = params["indicator"].get("num_export_days", "all")
 
     logger = get_structured_logger(
-        __name__, filename=params.get("log_filename"),
-        log_exceptions=params.get("log_exceptions", True))
+        __name__, filename=params["common"].get("log_filename"),
+        log_exceptions=params["common"].get("log_exceptions", True))
 
     # Pull GS data
-    dfs = pull_gs_data(params["path_to_bigquery_credentials"],
+    dfs = pull_gs_data(params["indicator"]["path_to_bigquery_credentials"],
                        export_start_date,
                        num_export_days)
     gmpr = geomap.GeoMapper()

google_symptoms/params.json.template

@@ -1,8 +1,12 @@
 {
-  "export_start_date": "2020-02-20",
-  "static_file_dir": "./static",
-  "export_dir": "./receiving",
-  "cache_dir": "./cache",
-  "path_to_bigquery_credentials": "credentials.json",
-  "log_exceptions": false
+  "common": {
+    "export_dir": "./receiving",
+    "log_exceptions": false
+  },
+  "indicator": {
+    "export_start_date": "2020-02-20",
+    "num_export_days": 14,
+    "path_to_bigquery_credentials": "credentials.json",
+    "static_file_dir": "./static"
+  }
 }

google_symptoms/tests/params.json.template

@@ -1,8 +1,11 @@
 {
-  "export_start_date": "2020-02-20",
-  "static_file_dir": "../static",
-  "export_dir": "./receiving",
-  "cache_dir": "./cache",
-  "num_export_days": 14,
-  "path_to_bigquery_credentials": ""
+  "common": {
+    "export_dir": "./receiving"
+  },
+  "indicator": {
+    "export_start_date": "2020-02-20",
+    "num_export_days": 14,
+    "path_to_bigquery_credentials": "",
+    "static_file_dir": "../static"
+  }
 }