Merge pull request #1770 from cmu-delphi/release/indicators_v0.3.29_utils_v0.3.8

Release covidcast-indicators 0.3.29

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.28
+current_version = 0.3.29
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

.github/workflows/python-ci.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, doctor_visits, dsew_community_profile, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot, usafacts]
+        packages: [_delphi_utils_python, changehc, claims_hosp, doctor_visits, dsew_community_profile, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot]
     defaults:
       run:
         working-directory: ${{ matrix.packages }}

Jenkinsfile

@@ -9,26 +9,43 @@
    - Keep in sync with '.github/workflows/python-ci.yml'.
    - TODO: #527 Get this list automatically from python-ci.yml at runtime.
  */
-def indicator_list = ["backfill_corrections", "changehc", "claims_hosp", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts", "dsew_community_profile", "doctor_visits"]
-def build_package = [:]
+
+def indicator_list = ["backfill_corrections", "changehc", "claims_hosp", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "dsew_community_profile", "doctor_visits"]
+def build_package_main = [:]
+def build_package_prod = [:]
 def deploy_staging = [:]
 def deploy_production = [:]
 
 pipeline {
     agent any
     stages {
-        stage('Build and Package') {
+        stage('Build and Package main') {
             when {
                 branch "main";
             }
             steps {
                 script {
                     indicator_list.each { indicator ->
-                        build_package[indicator] = {
-                            sh "jenkins/build-and-package.sh ${indicator}"
+                        build_package_main[indicator] = {
+                            sh "jenkins/build-and-package.sh ${indicator} main"
+                        }
+                    }
+                    parallel build_package_main
+                }
+            }
+        }
+        stage('Build and Package prod') {
+            when {
+                branch "prod";
+            }
+            steps {
+                script {
+                    indicator_list.each { indicator ->
+                        build_package_prod[indicator] = {
+                            sh "jenkins/build-and-package.sh ${indicator} prod"
                         }
                     }
-                    parallel build_package
+                    parallel build_package_prod
                 }
             }
         }

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.7
+current_version = 0.3.8
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.3.7"
+__version__ = "0.3.8"

_delphi_utils_python/delphi_utils/validator/params.json.template

@@ -1,7 +1,7 @@
 {
   "validation": {
     "global": {
-      "data_source": "usa-facts",
+      "data_source": "jhu-csse",
       "end_date": "2020-09-08",
       "span_length": 3,
       "suppressed_errors": [

_delphi_utils_python/delphi_utils/validator/static.py

@@ -90,28 +90,34 @@ def check_missing_date_files(self, daily_filenames, report):
         Returns:
             - None
         """
-        # Create set of all dates seen in CSV names.
-        unique_dates = {datetime.strptime(
-            daily_filename[0][0:8], '%Y%m%d').date() for daily_filename in daily_filenames}
-
-        # Diff expected and observed dates.
-        expected_dates = self.params.time_window.date_seq
-
-        if len(self.params.max_expected_lag) == 0:
-            max_expected_lag_overall = 10
-        else:
-            max_expected_lag_overall = max(self.params.max_expected_lag.values())
-
-        # Only check for date if it should definitely be present,
-        # i.e if it is more than max_expected_lag since the checking date
-        expected_dates = [date for date in expected_dates if
-            ((datetime.today().date() - date).days) > max_expected_lag_overall]
-        check_dateholes = list(set(expected_dates).difference(unique_dates))
-        check_dateholes.sort()
-
-        if check_dateholes:
+        # Check to see if there are any files in the export directory
+        # Validator will throw an error if the directory is empty, which can be suppressed
+        if len(daily_filenames) == 0:
             report.add_raised_error(
-                ValidationFailure("check_missing_date_files",
+                ValidationFailure("check_empty_filelist",
+                                  message="No files found in export directory"))
+        # Check for missing date only happens when files are found
+        else:
+            # Create set of all dates seen in CSV names.
+            unique_dates = {datetime.strptime(
+                daily_filename[0][0:8], '%Y%m%d').date() for daily_filename in daily_filenames}
+            # Diff expected and observed dates.
+            expected_dates = self.params.time_window.date_seq
+            if len(self.params.max_expected_lag) == 0:
+                max_expected_lag_overall = 10
+            else:
+                max_expected_lag_overall = max(self.params.max_expected_lag.values())
+
+            # Only check for date if it should definitely be present,
+            # i.e if it is more than max_expected_lag since the checking date
+            expected_dates = [date for date in expected_dates if
+                ((datetime.today().date() - date).days) > max_expected_lag_overall]
+            check_dateholes = list(set(expected_dates).difference(unique_dates))
+            check_dateholes.sort()
+
+            if check_dateholes:
+                report.add_raised_error(
+                    ValidationFailure("check_missing_date_files",
                                   message="Missing dates are observed; if these dates are already "
                                           "in the API they would not be updated"))
 

_delphi_utils_python/delphi_utils/validator/validate.py

@@ -58,6 +58,10 @@ def validate(self):
         frames_list = load_all_files(self.export_dir, self.time_window.start_date,
                                      self.time_window.end_date)
         self.static_validation.validate(frames_list, report)
-        all_frames = aggregate_frames(frames_list)
+        # Check if frames_list is empty before calling aggregate_frames
+        if len(frames_list) == 0:
+            all_frames = []
+        else:
+            all_frames = aggregate_frames(frames_list)
         self.dynamic_validation.validate(all_frames, report)
         return report

_delphi_utils_python/setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.3.7",
+    version="0.3.8",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

_delphi_utils_python/tests/validator/test_static.py

@@ -27,9 +27,29 @@ def test_empty_filelist(self):
         filenames = list()
         validator.check_missing_date_files(filenames, report)
 
+        assert len(report.raised_errors) == 1
+        assert report.raised_errors[0].check_name == "check_empty_filelist"
+
+    def test_missing_date_files(self):
+        params = {
+            "common": {
+                "data_source": "",
+                "span_length": 5,
+                "end_date": "2020-09-05",
+                "max_expected_lag": {"all": "1"}
+            }
+        }
+        validator = StaticValidator(params)
+        report = ValidationReport([])
+        filenames = [("20200901_county_signal_signal.csv", "match_obj"),
+                     ("20200903_county_signal_signal.csv", "match_obj"),
+                     ("20200904_county_signal_signal.csv", "match_obj"),
+                     ("20200905_county_signal_signal.csv", "match_obj")]
+        validator.check_missing_date_files(filenames, report)
         assert len(report.raised_errors) == 1
         assert report.raised_errors[0].check_name == "check_missing_date_files"
 
+
     def test_same_day(self):
         params = {
             "common": {

ansible/ansible-deploy-staging.yaml

@@ -6,7 +6,7 @@
   tasks:
     - name: Copy and unarchive the package into the indicators runtime host directory.
       unarchive:
-        src: "{{ jenkins_artifact_dir }}/{{ package }}"
+        src: "{{ jenkins_artifact_dir }}/{{ package_staging }}"
         dest: "{{ indicators_runtime_dir }}"
         owner: "{{ runtime_user }}"
         group: "{{ runtime_user }}"

ansible/ansible-deploy.yaml

@@ -6,7 +6,7 @@
   tasks:
     - name: Copy and unarchive the package into the indicators runtime host directory.
       unarchive:
-        src: "{{ jenkins_artifact_dir }}/{{ package }}"
+        src: "{{ jenkins_artifact_dir }}/{{ package_production }}"
         dest: "{{ indicators_runtime_dir }}"
         owner: "{{ runtime_user }}"
         group: "{{ runtime_user }}"

ansible/templates/claims_hosp-params-prod.json.j2

@@ -8,6 +8,8 @@
     "start_date": "2020-02-01",
     "end_date": null,
     "drop_date": null,
+    "backfill_dir": "/common/backfill/claims_hosp",
+    "backfill_merge_day": 0,
     "n_backfill_days": 70,
     "n_waiting_days": 3,
     "write_se": false,

ansible/templates/covid_act_now-params-prod.json.j2

@@ -28,11 +28,6 @@
         "sum_anosmia_ageusia_smoothed_search"
       ]
     },
-    "usa-facts": {
-      "max_age": 5,
-      "maintainers": ["U01AP8GSWG3","U01069KCRS7"],
-      "retired-signals": ["confirmed_7dav_cumulative_num", "confirmed_7dav_cumulative_prop", "deaths_7dav_cumulative_num", "deaths_7dav_cumulative_prop"]
-    },
     "jhu-csse": {
       "max_age": 2,
       "maintainers": ["U01AP8GSWG3","U01069KCRS7"],

ansible/templates/sir_complainsalot-params-prod.json.j2

@@ -6,7 +6,8 @@ runtime_user: "indicators"
 jenkins_user: "jenkins"
 jenkins_artifact_dir: "/var/lib/jenkins/artifacts"
 indicators_runtime_dir: "/home/{{ runtime_user }}/runtime"
-package: "{{ indicator }}.tar.gz" # {{ indicator }} is passed in from the Jenkins shell script wrapper.
+package_staging: "main_{{ indicator }}.tar.gz" # {{ indicator }} is passed in from the Jenkins shell script wrapper.
+package_production: "prod_{{ indicator }}.tar.gz" # {{ indicator }} is passed in from the Jenkins shell script wrapper.
 python_version: "3.8.2"
 pyenv_python_path: "/home/{{ runtime_user }}/.pyenv/versions/{{ python_version }}/bin/python"