Merge branch 'main' into backfill/claims_hosp

Author: nmdefries

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.19
+current_version = 0.3.28
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

.github/workflows/r-ci.yml renamed to .github/workflows/backfill-corr-ci.yml

@@ -6,7 +6,7 @@
 # See https://github.com/r-lib/actions/tree/master/examples#readme for
 # additional example workflows available for the R community.
 
-name: R facebook survey
+name: R backfill corrections
 
 on:
   push:
@@ -21,10 +21,10 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        r-version: [4.0]
+        r-version: [4.2.1]
     defaults:
       run:
-        working-directory: facebook/delphiFacebook
+        working-directory: backfill_corrections/delphiBackfillCorrection
 
     steps:
       - uses: actions/checkout@v2
@@ -34,7 +34,13 @@ jobs:
           r-version: ${{ matrix.r-version }}
       - name: Install linux dependencies
         run: |
-            sudo apt-get install libcurl4-openssl-dev
+            sudo apt-get install \
+              libcurl4-openssl-dev \
+              libgdal-dev \
+              libudunits2-dev \
+              libglpk-dev \
+              libharfbuzz-dev \
+              libfribidi-dev
       - name: Get date
         id: get-date
         run: |
@@ -43,17 +49,25 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-r-facebook-survey-${{ steps.get-date.outputs.date }}
+          key: ${{ runner.os }}-r-backfillcorr-${{ steps.get-date.outputs.date }}
           restore-keys: |
-            ${{ runner.os }}-r-facebook-survey-
+            ${{ runner.os }}-r-backfillcorr-
       - name: Install R dependencies
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           if ( !require("remotes") ) {
             install.packages("remotes")
           }
-          remotes::update_packages(c("rcmdcheck", "mockr", "remotes"), upgrade="always")
+          if ( !require("devtools") ) {
+            install.packages("devtools")
+          }
+          remotes::update_packages(c("rcmdcheck", "remotes", "devtools"), upgrade="always")
           dependency_list <- remotes::dev_package_deps(dependencies=TRUE)
           remotes::update_packages(dependency_list$package, upgrade="always")
+
+          devtools::install_github("cmu-delphi/covidcast", ref = "evalcast", subdir = "R-packages/evalcast")
+          devtools::install_github(repo="ryantibs/quantgen", subdir="quantgen")
         shell: Rscript {0}
       - name: Check
         run: |

.github/workflows/build-container-images.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        packages: [ facebook ]
+        packages: [ backfill_corrections ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

.github/workflows/create-release.yml

@@ -58,7 +58,13 @@ jobs:
         id: indicators
         run: |
           echo -n "::set-output name=version::"
-          bump2version --list ${{ github.event.inputs.versionName }} | grep ^new_version | sed -r s,"^.*=",,       
+          bump2version --list ${{ github.event.inputs.versionName }} | grep ^new_version | sed -r s,"^.*=",,
+      - name: Copy version to indicator directory
+        run: |
+          indicator_list=("changehc" "claims_hosp" "doctor_visits" "dsew_community_profile" "google_symptoms" "hhs_hosp" "jhu" "nchs_mortality" "nowcast" "quidel_covidtest" "sir_complainsalot")
+          for path in ${indicator_list[@]}; do
+            echo "current_version = ${{ steps.indicators.outputs.version }}" > $path/version.cfg
+          done
       - name: Create pull request into prod
         uses: peter-evans/create-pull-request@v3
         with:

.github/workflows/python-ci.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, doctor_visits, dsew_community_profile, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot, usafacts]
+        packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, doctor_visits, dsew_community_profile, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot]
     defaults:
       run:
         working-directory: ${{ matrix.packages }}

Jenkinsfile

@@ -9,26 +9,43 @@
    - Keep in sync with '.github/workflows/python-ci.yml'.
    - TODO: #527 Get this list automatically from python-ci.yml at runtime.
  */
-def indicator_list = ["changehc", "claims_hosp", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts", "dsew_community_profile"]
-def build_package = [:]
+
+def indicator_list = ["backfill_corrections", "changehc", "claims_hosp", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "dsew_community_profile", "doctor_visits"]
+def build_package_main = [:]
+def build_package_prod = [:]
 def deploy_staging = [:]
 def deploy_production = [:]
 
 pipeline {
     agent any
     stages {
-        stage('Build and Package') {
+        stage('Build and Package main') {
             when {
                 branch "main";
             }
             steps {
                 script {
                     indicator_list.each { indicator ->
-                        build_package[indicator] = {
-                            sh "jenkins/build-and-package.sh ${indicator}"
+                        build_package_main[indicator] = {
+                            sh "jenkins/build-and-package.sh ${indicator} main"
+                        }
+                    }
+                    parallel build_package_main
+                }
+            }
+        }
+        stage('Build and Package prod') {
+            when {
+                branch "prod";
+            }
+            steps {
+                script {
+                    indicator_list.each { indicator ->
+                        build_package_prod[indicator] = {
+                            sh "jenkins/build-and-package.sh ${indicator} prod"
                         }
                     }
-                    parallel build_package
+                    parallel build_package_prod
                 }
             }
         }

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.5
+current_version = 0.3.7
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.3.5"
+__version__ = "0.3.7"

_delphi_utils_python/delphi_utils/runner.py

@@ -1,6 +1,7 @@
 """Indicator running utilities."""
 import argparse as ap
 import importlib
+import os
 from typing import Any, Callable, Dict, Optional
 from .archive import ArchiveDiffer, archiver_from_params
 from .logger import get_structured_logger
@@ -40,6 +41,21 @@ def run_indicator_pipeline(indicator_fn:  Callable[[Params], None],
         name=indicator_fn.__module__,
         filename=params["common"].get("log_filename", None),
         log_exceptions=params["common"].get("log_exceptions", True))
+
+    #Get version and indicator name for startup
+    ind_name = indicator_fn.__module__.replace(".run", "")
+    #Check for version.cfg in indicator directory
+    if os.path.exists("version.cfg"):
+        with open("version.cfg") as ver_file:
+            current_version = "not found"
+            for line in ver_file:
+                if "current_version" in line:
+                    current_version = str.strip(line)
+                    current_version = current_version.replace("current_version = ", "")
+    #Logging - Starting Indicator
+        logger.info(f"Started {ind_name} with covidcast-indicators version {current_version}")
+    else: logger.info(f"Started {ind_name} without version.cfg")
+
     indicator_fn(params)
     validator = validator_fn(params)
     archiver = archiver_fn(params)

_delphi_utils_python/delphi_utils/validator/dynamic.py

@@ -5,7 +5,7 @@
 import re
 import pandas as pd
 import numpy as np
-from .errors import ValidationFailure, APIDataFetchError
+from .errors import ValidationFailure
 from .datafetcher import get_geo_signal_combos, threaded_api_calls
 from .utils import relative_difference_by_min, TimeWindow, lag_converter
 
@@ -116,7 +116,7 @@ def validate(self, all_frames, report):
             api_df_or_error = all_api_df[(geo_type, signal_type)]
 
             report.increment_total_checks()
-            if isinstance(api_df_or_error, APIDataFetchError):
+            if not isinstance(api_df_or_error, pd.DataFrame):
                 report.add_raised_error(api_df_or_error)
                 continue
 

_delphi_utils_python/delphi_utils/validator/params.json.template

@@ -1,7 +1,7 @@
 {
   "validation": {
     "global": {
-      "data_source": "usa-facts",
+      "data_source": "jhu-csse",
       "end_date": "2020-09-08",
       "span_length": 3,
       "suppressed_errors": [

_delphi_utils_python/setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.3.5",
+    version="0.3.7",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

ansible/ansible-deploy-staging.yaml

@@ -6,7 +6,7 @@
   tasks:
     - name: Copy and unarchive the package into the indicators runtime host directory.
       unarchive:
-        src: "{{ jenkins_artifact_dir }}/{{ package }}"
+        src: "{{ jenkins_artifact_dir }}/{{ package_staging }}"
         dest: "{{ indicators_runtime_dir }}"
         owner: "{{ runtime_user }}"
         group: "{{ runtime_user }}"

ansible/ansible-deploy.yaml

@@ -6,7 +6,7 @@
   tasks:
     - name: Copy and unarchive the package into the indicators runtime host directory.
       unarchive:
-        src: "{{ jenkins_artifact_dir }}/{{ package }}"
+        src: "{{ jenkins_artifact_dir }}/{{ package_production }}"
         dest: "{{ indicators_runtime_dir }}"
         owner: "{{ runtime_user }}"
         group: "{{ runtime_user }}"

ansible/templates/backfillcorr-params-prod.json.j2

@@ -0,0 +1,25 @@
+{
+  "ref_lag": 60,
+  "input_dir": "/common/backfill/",
+  "cache_dir": "./cache",
+  "testing_window": 1,
+  "training_days": 270,
+  "lag_pad":2,
+  "export_dir": "./receiving",
+  "geo_levels": ["state", "county"],
+  "value_types": ["count", "fraction"],
+  "num_col": "num",
+  "denom_col": "den",
+  "post": {
+    "aws_credentials": {
+      "aws_access_key_id": "{{ backfillcorr_aws_access_key_id }}",
+      "aws_secret_access_key": "{{ backfillcorr_aws_secret_access_key }}"
+    },
+    "bucket_name": "{{ backfillcorr_aws_bucket_name }}"
+  },
+  "gurobi": {
+    "GRB_LICENSEID": "{{ grb_licenseid }}",
+    "GRB_WLSACCESSID": "{{ grb_wlsaccessid }}",
+    "GRB_WLSSECRET": "{{ grb_wlssecret }}"
+  }
+}

ansible/templates/changehc-params-prod.json.j2

@@ -16,6 +16,8 @@
     "start_date": null,
     "end_date": null,
     "drop_date": null,
+    "backfill_dir": "/common/backfill/chng",
+    "backfill_merge_day": 0,
     "n_backfill_days": 60,
     "n_waiting_days": 3,
     "se": false,

ansible/templates/claims_hosp-params-prod.json.j2

@@ -1,6 +1,6 @@
 {
   "common": {
-    "export_dir": "./common/covidcast/receiving/claims_hosp",
+    "export_dir": "/common/covidcast/receiving/hospital-admissions/",
     "log_exceptions": false
   },
   "indicator": {
@@ -15,7 +15,7 @@
     "write_se": false,
     "obfuscated_prefix": "foo_obfuscated",
     "parallel": false,
-    "geos": ["state", "msa", "hrr", "county"],
+    "geos": ["state", "msa", "hrr", "county", "nation", "hhs"],
     "weekday": [true, false],
     "ftp_credentials": {
       "host": "{{ claims_hosp_ftp_host }}",

ansible/templates/doctor_visits-params-prod.json.j2

@@ -0,0 +1,45 @@
+{
+  "common": {
+    "export_dir": "./receiving",
+    "log_filename": "./doctor-visits.log"
+  },
+  "indicator": {
+    "input_file": "./input/SYNEDI_AGG_OUTPATIENT_18052020_1455CDT.csv.gz",
+    "drop_date": "",
+    "n_backfill_days": 70,
+    "n_waiting_days": 3,
+    "weekday": [true, false],
+    "se": false,
+    "obfuscated_prefix": "wip_XXXXX",
+    "parallel": false,
+    "ftp_credentials": {
+      "host": "{{ doctor_visits_ftp_host }}",
+      "user": "{{ doctor_visits_ftp_user }}",
+      "pass": "{{ doctor_visits_ftp_password }}",
+      "port": 2222
+    }
+  },
+  "validation": {
+    "common": {
+      "data_source": "doctor-visits",
+      "span_length": 14,
+      "min_expected_lag": {"all": "3"},
+      "max_expected_lag": {"all": "4"},
+      "dry_run": true,
+      "suppressed_errors": [
+      ]
+    },
+    "static": {
+      "minimum_sample_size": 5,
+      "missing_se_allowed": true,
+      "missing_sample_size_allowed": true
+    },
+    "dynamic": {
+      "ref_window_size": 7,
+      "smoothed_signals": [
+      "smoothed_cli",
+      "smoothed_adj_cli"
+      ]
+    }
+  }
+}