Merge pull request #1878 from cmu-delphi/release/indicators_v0.3.44_utils_v0.3.19

Release covidcast-indicators 0.3.44

Author: rzats

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.43
+current_version = 0.3.44
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

.github/ISSUE_TEMPLATE/data_quality_issue.md

@@ -3,7 +3,7 @@ name: Data quality issue
 about: Missing data, weird data, broken data 
 title: ''
 labels: 'data quality'
-assignees: 'krivard'
+assignees: 'nolangormley'
 ---
 
 **Actual Behavior:**

.github/workflows/create-delphi-utils-release.yml

@@ -61,7 +61,7 @@ jobs:
           bump2version --list ${{ github.event.inputs.versionName }} | grep ^new_version | sed -r s,"^.*=",,
       - name: Copy version to indicator directory
         run: |
-          indicator_list=("changehc" "claims_hosp" "doctor_visits" "dsew_community_profile" "google_symptoms" "hhs_hosp" "nchs_mortality" "nowcast" "quidel_covidtest" "sir_complainsalot")
+          indicator_list=("changehc" "claims_hosp" "doctor_visits" "google_symptoms" "hhs_hosp" "nchs_mortality" "nowcast" "quidel_covidtest" "sir_complainsalot")
           for path in ${indicator_list[@]}; do
             echo "current_version = ${{ steps.indicators.outputs.version }}" > $path/version.cfg
           done
@@ -72,8 +72,8 @@ jobs:
           base: prod
           title: Release covidcast-indicators ${{ steps.indicators.outputs.version }}
           labels: chore
-          reviewers: krivard
-          assignees: krivard
+          reviewers: melange396
+          assignees: melange396
           body: |
             Releasing:
             * covidcast-indicators ${{ steps.indicators.outputs.version }}

.github/workflows/create-release.yml

@@ -120,7 +120,7 @@ jobs:
           base: main
           title: 'chore: sync prod->main'
           labels: chore
-          reviewers: krivard
-          assignees: krivard
+          reviewers: melange396
+          assignees: melange396
           body: |
             Syncing Prod->Main.

.github/workflows/publish-release.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        packages: [_delphi_utils_python, changehc, claims_hosp, doctor_visits, dsew_community_profile, google_symptoms, hhs_hosp, nchs_mortality, nowcast, quidel_covidtest, sir_complainsalot]
+        packages: [_delphi_utils_python, changehc, claims_hosp, doctor_visits, google_symptoms, hhs_hosp, nchs_mortality, nowcast, quidel_covidtest, sir_complainsalot]
     defaults:
       run:
         working-directory: ${{ matrix.packages }}

.github/workflows/python-ci.yml

@@ -10,7 +10,7 @@
    - TODO: #527 Get this list automatically from python-ci.yml at runtime.
  */
 
-def indicator_list = ["backfill_corrections", "changehc", "claims_hosp", "google_symptoms", "hhs_hosp", "nchs_mortality", "quidel_covidtest", "sir_complainsalot", "dsew_community_profile", "doctor_visits"]
+def indicator_list = ["backfill_corrections", "changehc", "claims_hosp", "google_symptoms", "hhs_hosp", "nchs_mortality", "quidel_covidtest", "sir_complainsalot", "doctor_visits"]
 def build_package_main = [:]
 def build_package_prod = [:]
 def deploy_staging = [:]

.github/workflows/release-delphi-utils.yml

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.18
+current_version = 0.3.19
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

Jenkinsfile

@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.3.18"
+__version__ = "0.3.19"

_delphi_utils_python/.bumpversion.cfg

@@ -10,7 +10,6 @@
 import pandas as pd
 import numpy as np
 import covidcast
-from .. import read_params
 from .errors import APIDataFetchError, ValidationFailure
 
 FILENAME_REGEX = re.compile(
@@ -103,15 +102,13 @@ def load_csv(path):
         })
 
 
-def get_geo_signal_combos(data_source):
+def get_geo_signal_combos(data_source, api_key):
     """
     Get list of geo type-signal type combinations that we expect to see.
 
     Cross references based on combinations reported available by COVIDcast metadata.
     """
-    params = read_params()
-    assert "validation" in params
-    api_key = ("epidata", params["validation"]["common"]["api_credentials"])
+    api_key = ("epidata", api_key)
     # Maps data_source name with what's in the API, lists used in case of multiple names
     meta_response = requests.get("https://api.covidcast.cmu.edu/epidata/covidcast/meta",
                                  auth=api_key)
@@ -166,14 +163,13 @@ def fetch_api_reference(data_source, start_date, end_date, geo_type, signal_type
         api_df = covidcast.signal(
             data_source, signal_type, start_date, end_date, geo_type)
 
-    if not isinstance(api_df, pd.DataFrame):
-        custom_msg = "Error fetching data from " + str(start_date) + \
-                     " to " + str(end_date) + \
-                     " for data source: " + data_source + \
-                     ", signal type: " + signal_type + \
-                     ", geo type: " + geo_type
+    error_context = f"when fetching reference data from {start_date} to {end_date} " +\
+        f"for data source: {data_source}, signal type: {signal_type}, geo type: {geo_type}"
 
-        raise APIDataFetchError(custom_msg)
+    if api_df is None:
+        raise APIDataFetchError("Error: no API data was returned " + error_context)
+    if not isinstance(api_df, pd.DataFrame):
+        raise APIDataFetchError("Error: API return value was not a dataframe " + error_context)
 
     column_names = ["geo_id", "val",
                     "se", "sample_size", "time_value"]
@@ -220,12 +216,15 @@ def get_one_api_df(data_source, min_date, max_date,
     dict_lock.release()
 
 
-def threaded_api_calls(data_source, min_date, max_date, geo_signal_combos, n_threads=32):
+MAX_ALLOWED_THREADS = 32
+
+def threaded_api_calls(data_source, min_date, max_date,
+                       geo_signal_combos, n_threads=MAX_ALLOWED_THREADS):
     """Get data from API for all geo-signal combinations in a threaded way."""
-    if n_threads > 32:
-        n_threads = 32
-        print("Warning: Don't run more than 32 threads at once due "
-                + "to API resource limitations")
+    if n_threads > MAX_ALLOWED_THREADS:
+        n_threads = MAX_ALLOWED_THREADS
+        warnings.warn("Warning: instead of requested thread count, using " + \
+            f"only {MAX_ALLOWED_THREADS} threads due to API resource limitations")
 
     output_dict = dict()
     dict_lock = threading.Lock()

_delphi_utils_python/delphi_utils/__init__.py

@@ -5,6 +5,7 @@
 import re
 import pandas as pd
 import numpy as np
+import covidcast
 from .errors import ValidationFailure
 from .datafetcher import get_geo_signal_combos, threaded_api_calls
 from .utils import relative_difference_by_min, TimeWindow, lag_converter
@@ -14,12 +15,14 @@ class DynamicValidator:
     """Class for validation of static properties of individual datasets."""
 
     @dataclass
-    class Parameters:
+    class Parameters: # pylint: disable=too-many-instance-attributes
         """Configuration parameters."""
 
         # data source name, one of
         # https://cmu-delphi.github.io/delphi-epidata/api/covidcast_signals.html
         data_source: str
+        # COVIDcast API key
+        api_key: str
         # span of time over which to perform checks
         time_window: TimeWindow
         # date that this df_to_test was generated; typically 1 day after the last date in df_to_test
@@ -48,6 +51,7 @@ def __init__(self, params):
 
         self.params = self.Parameters(
             data_source=common_params["data_source"],
+            api_key = common_params["api_credentials"],
             time_window=TimeWindow.from_params(common_params["end_date"],
                                                common_params["span_length"]),
             generation_date=date.today(),
@@ -74,8 +78,12 @@ def validate(self, all_frames, report):
         # Get 14 days prior to the earliest list date
         outlier_lookbehind = timedelta(days=14)
 
+        # Authenticate API
+        covidcast.use_api_key(self.params.api_key)
+
         # Get all expected combinations of geo_type and signal.
-        geo_signal_combos = get_geo_signal_combos(self.params.data_source)
+        geo_signal_combos = get_geo_signal_combos(self.params.data_source,
+                                                  api_key = self.params.api_key)
 
         all_api_df = threaded_api_calls(self.params.data_source,
                                         self.params.time_window.start_date - outlier_lookbehind,

_delphi_utils_python/delphi_utils/validator/datafetcher.py

@@ -27,7 +27,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.3.18",
+    version="0.3.19",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

_delphi_utils_python/delphi_utils/validator/dynamic.py

@@ -54,7 +54,7 @@ def raise_for_status(self):
     def test_bad_api_key(self, **kwargs):
         kwargs["mock_requests"].get("https://api.covidcast.cmu.edu/epidata/covidcast/meta", status_code=429)
         with pytest.raises(HTTPError):
-            get_geo_signal_combos("chng")
+            get_geo_signal_combos("chng", api_key="")
 
     @mock.patch('requests.get', side_effect=mocked_requests_get)
     @mock.patch("covidcast.metadata")
@@ -78,11 +78,11 @@ def test_get_geo_signal_combos(self, mock_metadata, mock_get):
                                                                 "hrr", "msa", "msa",
                                                                 "state"]
                                                   })
-        assert set(get_geo_signal_combos("chng")) == set(
+        assert set(get_geo_signal_combos("chng", api_key="")) == set(
             [("state", "smoothed_outpatient_cli"),
              ("state", "smoothed_outpatient_covid"),
              ("county", "smoothed_outpatient_covid")])
-        assert set(get_geo_signal_combos("covid-act-now")) == set(
+        assert set(get_geo_signal_combos("covid-act-now", api_key="")) == set(
             [("hrr", "pcr_specimen_positivity_rate"),
              ("msa", "pcr_specimen_positivity_rate"),
              ("msa", "pcr_specimen_total_tests")])
@@ -138,7 +138,8 @@ def mock_signal_return_fn(unused_data_source, signal_type, unused_start_date,
             ("state", "b"): ValidationFailure("api_data_fetch_error",
                                               geo_type="state",
                                               signal="b",
-                                             message="Error fetching data from 2020-03-10 "
+                                             message="Error: no API data was returned when "
+                                             "fetching reference data from 2020-03-10 "
                                              "to 2020-06-10 for data source: "
                                              "source, signal type: b, geo type: state")
         }