Merge pull request #807 from sgsmob/jhu2

Refactor parameter hierarchy in JHU indicator

Author: krivard

ansible/templates/jhu-params-prod.json.j2

@@ -1,13 +1,19 @@
 {
-  "export_start_date": "2020-02-20",
-  "static_file_dir": "./static",
-  "export_dir": "/common/covidcast/receiving/jhu-csse/",
-  "cache_dir": "./cache",
-  "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv",
-  "aws_credentials": {
-    "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
-    "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+  "common": {
+    "export_dir": "/common/covidcast/receiving/jhu-csse/",
+    "log_filename": "/var/log/indicators/jhu.log"
   },
-  "bucket_name": "delphi-covidcast-indicator-output",
-  "log_filename": "/var/log/indicators/jhu.log"
+  "indicator": {
+    "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv",
+    "export_start_date": "2020-02-20",
+    "static_file_dir": "./static"
+  },
+  "archive": {
+    "aws_credentials": {
+      "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
+      "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+    },
+    "bucket_name": "delphi-covidcast-indicator-output",
+    "cache_dir": "./cache"
+  }
 }

jhu/delphi_jhu/run.py

@@ -69,21 +69,20 @@ def run_module():
     csv_export_count = 0
     oldest_final_export_date = None
     params = read_params()
-    export_start_date = params["export_start_date"]
-    export_dir = params["export_dir"]
-    base_url = params["base_url"]
-    cache_dir = params["cache_dir"]
+    export_start_date = params["indicator"]["export_start_date"]
+    export_dir = params["common"]["export_dir"]
+    base_url = params["indicator"]["base_url"]
     logger = get_structured_logger(
-        __name__, filename=params.get("log_filename"),
-        log_exceptions=params.get("log_exceptions", True))
+        __name__, filename=params["common"].get("log_filename"),
+        log_exceptions=params["common"].get("log_exceptions", True))
 
-    if len(params["bucket_name"]) > 0:
+    if "archive" in params:
         arch_diff = S3ArchiveDiffer(
-            cache_dir,
+            params["archive"]["cache_dir"],
             export_dir,
-            params["bucket_name"],
+            params["archive"]["bucket_name"],
             "jhu",
-            params["aws_credentials"],
+            params["archive"]["aws_credentials"],
         )
         arch_diff.update_cache()
     else:

jhu/params.json.template

@@ -1,13 +1,19 @@
 {
-  "export_start_date": "2020-02-20",
-  "static_file_dir": "./static",
-  "export_dir": "./receiving",
-  "cache_dir": "./cache",
-  "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv",
-  "aws_credentials": {
-    "aws_access_key_id": "",
-    "aws_secret_access_key": ""
+  "common": {
+    "export_dir": "./receiving",
+    "log_exceptions": false
   },
-  "bucket_name": "",
-  "log_exceptions": false
+  "indicator": {
+    "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv",
+    "export_start_date": "2020-02-20",
+    "static_file_dir": "./static"
+  },
+  "archive": {
+    "aws_credentials": {
+      "aws_access_key_id": "",
+      "aws_secret_access_key": ""
+    },
+    "bucket_name": "",
+    "cache_dir": "./cache"
+  }
 }

jhu/tests/conftest.py

@@ -22,9 +22,9 @@ def run_as_module():
     with mock_s3():
         # Create the fake bucket we will be using
         params = read_params()
-        aws_credentials = params["aws_credentials"]
+        aws_credentials = params["archive"]["aws_credentials"]
         s3_client = Session(**aws_credentials).client("s3")
-        s3_client.create_bucket(Bucket=params["bucket_name"])
+        s3_client.create_bucket(Bucket=params["archive"]["bucket_name"])
 
         run_module()
 

jhu/tests/params.json.template

@@ -1,12 +1,18 @@
 {
-    "export_start_date": "2020-03-03",
-    "static_file_dir": "../static",
-    "export_dir": "./receiving",
-    "cache_dir": "./cache",
+  "common": {
+    "export_dir": "./receiving"
+  },
+  "indicator": {
     "base_url": "test_data/small_{metric}.csv",
+    "export_start_date": "2020-03-03",
+    "static_file_dir": "../static"
+  },
+  "archive": {
     "aws_credentials": {
       "aws_access_key_id": "FAKE_TEST_ACCESS_KEY_ID",
       "aws_secret_access_key": "FAKE_TEST_SECRET_ACCESS_KEY"
     },
-    "bucket_name": "test-bucket"
+    "bucket_name": "test-bucket",
+    "cache_dir": "./cache"
+  }
 }