Merge pull request #229 from cmu-delphi/dev/rebase-jhu-deploy

Fix JHU deployment history

Author: krivard

ansible/files/jhu-params-prod.json renamed to ansible/templates/jhu-params-prod.json.j2

@@ -3,5 +3,10 @@
   "static_file_dir": "./static",
   "export_dir": "/common/covidcast/receiving/jhu-csse/",
   "cache_dir": "./cache",
-  "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv"
+  "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv",
+  "aws_credentials": {
+    "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
+    "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+  },
+  "bucket_name": "delphi-covidcast-indicator-output"
 }

ansible/vars.yaml

@@ -9,3 +9,6 @@ pyenv_python_path: "/home/{{ runtime_user }}/.pyenv/versions/{{ python_version }
 
 # Indicators variables.
 google_health_api_key: "{{ vault_google_health_api_key }}"
+delphi_aws_access_key_id: "{{ vault_delphi_aws_access_key_id }}"
+delphi_aws_secret_access_key: "{{ vault_delphi_aws_secret_access_key }}"
+

ansible/vault.yaml

@@ -1,9 +1,16 @@
 $ANSIBLE_VAULT;1.1;AES256
-35386139306136373731386533393535343530623035366532343334326430313839336334646363
-6630646235616664386630623565363262643965376563660a386639663436616635323630356430
-35613139666439363161613232346463366637623238623963623364366166356131346363656432
-3831636539306230660a656430623166663262316230333137613633333337623338393837643331
-34306335636231366135383163336435613130396265333434623361376531396138353163653265
-33363963363032326164323161323036326436616333303738373866623335623664316565653461
-63646536633161623937363062663031353734303539343332633365326333353537316336383461
-64353265303234346633
+66386163643862646634343162646465663762643034303563333833633661333932646164656462
+6166646131623132393238336263623562373065643633310a663232373237396361623462613333
+62373663383565623263306539636431623230633065626363666531366662363065343066363031
+3738616663336665340a326138333634306137363837396366303861663064326333613662656630
+62306331646637326637363766366237663037306665343761643263646663316535343561623137
+63313365653535393639626465343232396261643239303430383138633135346466323834336665
+33633064353034613836313265613466623961373565363835343430373138376336363966316365
+35663664396436313432376264316663326130306134326231303234393561643436623039613136
+63366638396262383762383336643930343661636461646162653734336334306239383132643435
+39333665643738643966356431333830646561353263353063326330643731616130396466343339
+39346437653063303336626663623835613938633834396430353634383366386237353862643766
+37393738353231666565303031393839306463373461393761653866653330646534393832303264
+30323038646166366465396235623731343539313633326539663966333437623733626131653437
+62326632656462383835656235373664366566343866383938343639613737623631616231616135
+633863383761366461363532353137323936

google_health/delphi_google_health/run.py

@@ -24,7 +24,7 @@ def run_module():
     testing purposes).
     """
 
-    #  read parameters
+    # read parameters
     params = read_params()
     ght_key = params["ght_key"]
     start_date = params["start_date"]
@@ -51,7 +51,7 @@ def run_module():
     # setup class to handle API calls
     ght = GoogleHealthTrends(ght_key=ght_key)
 
-    #  read data frame version of the data
+    # read data frame version of the data
     df_state = get_counts_states(
         ght, start_date, end_date, static_dir=static_dir, cache_dir=cache_dir
     )
@@ -64,7 +64,7 @@ def run_module():
 
     for signal in signal_names:
         if signal.endswith(SMOOTHED):
-            #  export each geographic region, with both smoothed and unsmoothed data
+            # export each geographic region, with both smoothed and unsmoothed data
             export_csv(df_state, STATE, signal, smooth=True, receiving_dir=export_dir)
             export_csv(df_dma, DMA, signal, smooth=True, receiving_dir=export_dir)
             export_csv(df_hrr, HRR, signal, smooth=True, receiving_dir=export_dir)

jhu/params.json.template

@@ -6,6 +6,7 @@
   "base_url": "https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_{metric}_US.csv",
   "aws_credentials": {
     "aws_access_key_id": "",
-    "aws_secret_access_key": "",
-  }
+    "aws_secret_access_key": ""
+  },
+  "bucket_name": ""
 }

jhu/tests/conftest.py

@@ -1,10 +1,13 @@
 # -*- coding: utf-8 -*-
 
+from boto3 import Session
+from moto import mock_s3
 import pytest
 
 from os import listdir, remove
 from os.path import join
 
+from delphi_utils import read_params
 from delphi_jhu.run import run_module
 
 
@@ -14,4 +17,11 @@ def run_as_module():
     for fname in listdir("receiving"):
         remove(join("receiving", fname))
 
-    run_module()
+    with mock_s3():
+        # Create the fake bucket we will be using
+        params = read_params()
+        aws_credentials = params["aws_credentials"]
+        s3_client = Session(**aws_credentials).client("s3")
+        s3_client.create_bucket(Bucket=params["bucket_name"])
+
+        run_module()

jhu/tests/params.json.template

@@ -3,5 +3,10 @@
     "static_file_dir": "../static",
     "export_dir": "./receiving",
     "cache_dir": "./cache",
-    "base_url": "test_data/small_{metric}.csv"
+    "base_url": "test_data/small_{metric}.csv",
+    "aws_credentials": {
+      "aws_access_key_id": "FAKE_TEST_ACCESS_KEY_ID",
+      "aws_secret_access_key": "FAKE_TEST_SECRET_ACCESS_KEY"
+    },
+    "bucket_name": "test-bucket"
 }