Merge pull request #1865 from cmu-delphi/release/indicators_v0.3.42_utils_v0.3.17

Release covidcast-indicators 0.3.42

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.41
+current_version = 0.3.42
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.16
+current_version = 0.3.17
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.3.16"
+__version__ = "0.3.17"

_delphi_utils_python/delphi_utils/validator/datafetcher.py

@@ -111,9 +111,10 @@ def get_geo_signal_combos(data_source):
     Cross references based on combinations reported available by COVIDcast metadata.
     """
     # Maps data_source name with what's in the API, lists used in case of multiple names
-
+    meta_response = requests.get("https://api.covidcast.cmu.edu/epidata/covidcast/meta")
+    meta_response.raise_for_status()
     source_signal_mappings = {i['source']:i['db_source'] for i in
-        requests.get("https://api.covidcast.cmu.edu/epidata/covidcast/meta").json()}
+        meta_response.json()}
     meta = covidcast.metadata()
     source_meta = meta[meta['data_source'] == data_source]
     # Need to convert np.records to tuples so they are hashable and can be used in sets and dicts.
@@ -139,6 +140,7 @@ def get_geo_signal_combos(data_source):
                 epidata_signal = requests.get(
                     "https://api.covidcast.cmu.edu/epidata/covidcast/meta",
                     params={'signal': f"{src}:{sig}"})
+                epidata_signal.raise_for_status()
                 # Not an active signal
                 active_status = [val['active'] for i in epidata_signal.json()
                     for val in i['signals']]

_delphi_utils_python/delphi_utils/validator/run.py

@@ -5,6 +5,7 @@
 when the module is run with `python -m delphi_utils.validator`.
 """
 import argparse as ap
+import covidcast
 from .. import read_params, get_structured_logger
 from .validate import Validator
 
@@ -17,6 +18,7 @@ def run_module():
     args = parser.parse_args()
     params = read_params()
     assert "validation" in params
+    covidcast.use_api_key(params["validation"]["common"]["api_credentials"])
     dry_run_param = params["validation"]["common"].get("dry_run", False)
     params["validation"]["common"]["dry_run"] = args.dry_run or dry_run_param
     validator = Validator(params)

_delphi_utils_python/setup.py

@@ -19,14 +19,15 @@
     "pylint==2.8.3",
     "pytest",
     "pytest-cov",
+    "requests-mock",
     "slackclient",
     "structlog",
     "xlrd"
 ]
 
 setup(
     name="delphi_utils",
-    version="0.3.16",
+    version="0.3.17",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

_delphi_utils_python/tests/validator/test_datafetcher.py

@@ -4,13 +4,17 @@
 import mock
 import numpy as np
 import pandas as pd
+import pytest
+from requests.exceptions import HTTPError
+import requests_mock
 from delphi_utils.validator.datafetcher import (FILENAME_REGEX,
                                                 make_date_filter,
                                                 get_geo_signal_combos,
                                                 threaded_api_calls)
 from delphi_utils.validator.errors import ValidationFailure
 
 
+
 class TestDataFetcher:
     """Tests for various data fetching utilities."""
     def test_make_date_filter(self):
@@ -24,20 +28,34 @@ def test_make_date_filter(self):
     # Solution from https://stackoverflow.com/questions/15753390/
     #how-can-i-mock-requests-and-the-response
     def mocked_requests_get(*args, **kwargs):
+        # TODO #1863: convert to requests_mock
         class MockResponse:
             def __init__(self, json_data, status_code):
                 self.json_data = json_data
                 self.status_code = status_code
 
             def json(self):
                 return self.json_data
+
+            def raise_for_status(self):
+                if self.status_code != 200:
+                    raise HTTPError()
         if len(kwargs) == 0:
             return MockResponse([{'source': 'chng', 'db_source': 'chng'},
                 {'source': 'covid-act-now', 'db_source': 'covid-act-now'}], 200)
         elif kwargs["params"] == {'signal': 'chng:inactive'}:
             return MockResponse([{"signals": [{"active": False}]}], 200)
         else:
             return MockResponse([{"signals": [{"active": True}]}], 200)
+
+    # the `kw` approach is needed here because otherwise pytest thinks the 
+    # requests_mock arg is supposed to be a fixture
+    @requests_mock.Mocker(kw="mock_requests")
+    def test_bad_api_key(self, **kwargs):
+        kwargs["mock_requests"].get("https://api.covidcast.cmu.edu/epidata/covidcast/meta", status_code=429)
+        with pytest.raises(HTTPError):
+            get_geo_signal_combos("chng")
+
     @mock.patch('requests.get', side_effect=mocked_requests_get)
     @mock.patch("covidcast.metadata")
     def test_get_geo_signal_combos(self, mock_metadata, mock_get):

ansible/templates/changehc-params-prod.json.j2

@@ -37,6 +37,7 @@
   "validation": {
     "common": {
       "data_source": "chng",
+      "api_credentials": "{{ validation_api_key }}",
       "span_length": 14,
       "min_expected_lag": {"all": "4"},
       "max_expected_lag": {"all": "6"},

ansible/templates/claims_hosp-params-prod.json.j2

@@ -28,6 +28,7 @@
   "validation": {
     "common": {
       "data_source": "hospital-admissions",
+      "api_credentials": "{{ validation_api_key }}",
       "span_length": 14,
       "min_expected_lag": {"all": "3"},
       "max_expected_lag": {"all": "4"},

ansible/templates/doctor_visits-params-prod.json.j2

@@ -22,6 +22,7 @@
   "validation": {
     "common": {
       "data_source": "doctor-visits",
+      "api_credentials": "{{ validation_api_key }}",
       "span_length": 14,
       "min_expected_lag": {"all": "3"},
       "max_expected_lag": {"all": "4"},

ansible/templates/dsew_community_profile-params-prod.json.j2

@@ -18,6 +18,7 @@
   "validation": {
     "common": {
       "data_source": "dsew-cpr",
+      "api_credentials": "{{ validation_api_key }}",
       "span_length": 14,
       "min_expected_lag": {"all": "3"},
       "max_expected_lag": {"all": "9"},

ansible/templates/google_symptoms-params-prod.json.j2

@@ -6,6 +6,7 @@
   "indicator": {
     "export_start_date": "2020-02-20",
     "num_export_days": null,
+    "api_credentials": "{{ google_symptoms_api_key }}",
     "bigquery_credentials": {
       "type": "{{ google_symptoms_account_type }}",
       "project_id": "{{ google_symptoms_project_id }}",
@@ -22,6 +23,7 @@
   "validation": {
     "common": {
       "data_source": "google-symptoms",
+      "api_credentials": "{{ validation_api_key }}",
       "validation_failure_dir": "./validation_failures",
       "span_length": 14,
       "min_expected_lag": {"all": "3"},

ansible/templates/hhs_hosp-params-prod.json.j2

@@ -6,6 +6,7 @@
   "validation": {
     "common": {
       "data_source": "hhs",
+      "api_credentials": "{{ validation_api_key }}",
       "span_length": 14,
       "min_expected_lag": {"all": "1"},
       "max_expected_lag": {"all": "7"},

ansible/templates/quidel_covidtest-params-prod.json.j2

@@ -25,6 +25,7 @@
   "validation": {
     "common": {
       "data_source": "quidel",
+      "api_credentials": "{{ validation_api_key }}",
       "span_length": 14,
       "min_expected_lag": {"all": "5"},
       "max_expected_lag": {"all": "5"},

ansible/templates/sir_complainsalot-params-prod.json.j2

@@ -1,6 +1,7 @@
 {
   "log_filename": "/var/log/indicators/sircal.log",
   "slack_token": "{{ sir_complainsalot_slack_token }}",
+  "api_credentials": "{{ sir_complainsalot_api_key }}",
   "sources": {
     "doctor-visits": {
       "max_age": 5,

ansible/vars.yaml

@@ -14,6 +14,8 @@ pyenv_python_path: "/home/{{ runtime_user }}/.pyenv/versions/{{ python_version }
 #
 # Indicators variables
 #
+# Validation
+validation_api_key: "{{ vault_validator_api_key }}"
 # Google Health
 google_health_api_key: "{{ vault_google_health_api_key }}"
 delphi_aws_access_key_id: "{{ vault_delphi_aws_access_key_id }}"
@@ -52,6 +54,7 @@ doctor_visits_midas_password: "{{ vault_doctor_visits_midas_password }}"
 nchs_mortality_token: "{{ vault_nchs_mortality_token }}"
 
 # SirCAL
+sir_complainsalot_api_key: "{{ vault_sir_complainsalot_api_key }}"
 sir_complainsalot_slack_token: "{{ vault_sir_complainsalot_slack_token }}"
 
 # Survey
@@ -62,6 +65,7 @@ fb_survey_sftp_password: "{{ vault_fb_survey_sftp_password }}"
 fb_survey_sftp_user: "{{ vault_fb_survey_sftp_user }}"
 
 ## Google Symptoms
+google_symptoms_api_key: "{{ vault_google_symptoms_api_key }}"
 google_symptoms_account_type: "{{ vault_google_symptoms_account_type }}"
 google_symptoms_project_id: "{{ vault_google_symptoms_project_id }}"
 google_symptoms_private_key_id: "{{ vault_google_symptoms_private_key_id }}"