Merge branch 'add-hhs-to-utils' into dv-hhs-nation

Author: chinandrew

.github/CONTRIBUTING.md

@@ -61,7 +61,7 @@ git checkout -b dev-my-feature-branch
 
 ### Creating your indicator
 
-Create a directory for your new indicator by making a copy of `_template_r` or `_template_python` depending on the programming language you intend to use. The template copies of `README.md` and `REVIEW.md` include the minimum requirements for code structure, documentation, linting, testing, and method of configuration. Beyond that, we don't have any established restrictions on implementation; you can look at other existing indicators see some examples of code layout, organization, and general approach.
+Create a directory for your new indicator by making a copy of `_template_r` or `_template_python` depending on the programming language you intend to use. If using Python, add the name of the directory to the list found in `jobs > build > strategy > matrix > packages` in `.github/workflows/python-ci.yml`, which will enable automated checks for your indicator when you make PRs. The template copies of `README.md` and `REVIEW.md` include the minimum requirements for code structure, documentation, linting, testing, and method of configuration. Beyond that, we don't have any established restrictions on implementation; you can look at other existing indicators see some examples of code layout, organization, and general approach.
 
 - Consult your peers with questions! :handshake:
 

.github/ISSUE_TEMPLATE/feature_release.md

@@ -15,13 +15,14 @@ assignees: 'benjaminysmith'
 
 <!-- relevant for most work -->
 
-- [ ] API [documentation](https://github.com/cmu-delphi/delphi-epidata/tree/main/docs/api) and/or [changelog](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast_changelog.md)
+- [ ] API [documentation](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast-signals/_source-template.md) and/or [changelog](https://github.com/cmu-delphi/delphi-epidata/blob/main/docs/api/covidcast_changelog.md)
 - [ ] API mailing list notification
 
 <!-- relevant for new signals -->
 
 - [ ] Statistical review (usually [correlations](https://github.com/cmu-delphi/covidcast/tree/main/docs/R-notebooks))
 - [ ] Signal / source name review (usually [Roni](https://docs.google.com/document/d/10hGd4Evce4lJ4VkWaQEKFQxvmw2P4xyYGtIAWF52Sf8/edit?usp=sharing))
+- [ ] Verify licensing terms and any applicable DUA restrictions
 
 <!-- relevant for new map signals -->
 

.github/pull_request_template.md

@@ -0,0 +1,10 @@
+### Description
+Type of change (bug fix, new feature, etc), brief description, and motivation for these changes.
+
+### Changelog
+Itemize code/test/documentation changes and files added/removed.
+- change1
+- change2
+
+### Fixes 
+- Fixes #(issue)

.github/workflows/python-ci.yml

@@ -0,0 +1,42 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ main, prod ]
+  pull_request:
+    types: [ opened, synchronize, reopened, ready_for_review ]
+    branches: [ main, prod ]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        packages: [_delphi_utils_python, cdc_covidnet, changehc, claims_hosp, combo_cases_and_deaths, google_symptoms, jhu, nchs_mortality, quidel, quidel_covidtest, safegraph, safegraph_patterns, usafacts]
+    defaults:
+      run:
+        working-directory: ${{ matrix.packages }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install testing dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pylint pytest pydocstyle wheel
+    - name: Install
+      run: |
+        make install
+    - name: Lint
+      if: ${{ matrix.packages != 'claims_hosp' }}
+      run: |
+        make lint
+    - name: Test
+      run: |
+        make test

.gitignore

@@ -126,3 +126,7 @@ venv.bak/
 .retry
 .indicators-ansible-vault-pass
 indicators-ansible-vault-pass
+
+# testing_utils
+testing_utils/cache
+testing_utils/*.csv

Jenkinsfile

@@ -1,81 +1,73 @@
 #!groovy
 
-// import shared library: https://github.com/cmu-delphi/jenkins-shared-library
+// Import shared lib.
 @Library('jenkins-shared-library') _
 
-pipeline {
+/*
+   Declare variables.
+   - indicator_list should contain all the indicators to handle in the pipeline.
+   - Keep in sync with '.github/workflows/python-ci.yml'.
+   - TODO: #527 Get this list automatically from python-ci.yml at runtime.
+ */
+def indicator_list = ["cdc_covidnet", "changehc", "claims_hosp", "combo_cases_and_deaths", "google_symptoms", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph", "safegraph_patterns", "usafacts"]
+def build_package = [:]
+def deploy_staging = [:]
+def deploy_production = [:]
 
+pipeline {
     agent any
-
     stages {
-
-        stage ("Environment") {            
+        stage('Build and Package') {
             when {
-                anyOf {
-                    branch "deploy-*";
-                    changeRequest target: "deploy-*", comparator: "GLOB"
-                }
+                branch "main";
             }
             steps {
                 script {
-                    // Get the indicator name from the pipeline env.
-                    if ( env.CHANGE_TARGET ) {
-                        INDICATOR = env.CHANGE_TARGET.replaceAll("deploy-", "")
-                    }
-                    else if ( env.BRANCH_NAME ) {
-                        INDICATOR = env.BRANCH_NAME.replaceAll("deploy-", "")
-                    }
-                    else {
-                        INDICATOR = ""
+                    indicator_list.each { indicator ->
+                        build_package[indicator] = {
+                            sh "jenkins/build-and-package.sh ${indicator}"
+                        }
                     }
-                } 
-            }
-        }
-
-        stage('Build') {
-            when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
-            }
-            steps {
-                sh "jenkins/${INDICATOR}-jenkins-build.sh"
-            }
-        }
-
-        stage('Test') {
-            when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
-            }
-            steps {
-                sh "jenkins/${INDICATOR}-jenkins-test.sh"
+                    parallel build_package
+                }
             }
         }
-        
-        stage('Package') {
+        stage('Deploy staging') {
             when {
-                changeRequest target: "deploy-*", comparator: "GLOB"
+                branch "main";
             }
             steps {
-                sh "jenkins/${INDICATOR}-jenkins-package.sh"
+                script {
+                    indicator_list.each { indicator ->
+                        deploy_staging[indicator] = {
+                            sh "jenkins/deploy-staging.sh ${indicator}"
+                        }
+                    }
+                    parallel deploy_staging
+                }
+                sh "jenkins/deploy-staging-api-match-list.sh"
             }
         }
-
-        stage('Deploy') {
+        stage('Deploy production') {
             when {
-                branch "deploy-*"
+                branch "prod";
             }
             steps {
-                sh "jenkins/${INDICATOR}-jenkins-deploy.sh"
+                script {
+                    indicator_list.each { indicator ->
+                        deploy_production[indicator] = {
+                            sh "jenkins/deploy-production.sh ${indicator}"
+                        }
+                    }
+                    parallel deploy_production
+                }
             }
         }
     }
-
     post {
         always {
             script {
-                /*
-                Use slackNotifier.groovy from shared library and provide current
-                build result as parameter.
-                */   
+                // Use slackNotifier.groovy from shared lib.
                 slackNotifier(currentBuild.currentResult)
             }
         }

_delphi_utils_python/.pylintrc

@@ -1,8 +1,22 @@
-[DESIGN]
 
-min-public-methods=1
+[MESSAGES CONTROL]
 
+disable=logging-format-interpolation,
+    too-many-locals,
+    too-many-arguments,
+    # Allow pytest functions to be part of a class.
+    no-self-use,
+    # Allow pytest classes to have one test.
+    too-few-public-methods
 
-[MESSAGES CONTROL]
+[BASIC]
+
+# Allow arbitrarily short-named variables.
+variable-rgx=[a-z_][a-z0-9_]*
+argument-rgx=[a-z_][a-z0-9_]*
+attr-rgx=[a-z_][a-z0-9_]*
+
+[DESIGN]
 
-disable=R0801, C0330, E1101, E0611, C0114, C0116, C0103, R0913, R0914, W0702
+# Don't complain about pytest "unused" arguments.
+ignored-argument-names=(_.*|run_as_module)

_delphi_utils_python/Makefile

@@ -0,0 +1,24 @@
+.PHONY = venv, lint, test, clean
+
+dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*')
+
+venv:
+	python3.8 -m venv env
+
+install: venv
+	. env/bin/activate; \
+	pip install wheel ; \
+	pip install -e .
+
+lint:
+	. env/bin/activate; \
+	pylint $(dir); \
+	pydocstyle $(dir)
+
+test:
+	. env/bin/activate ;\
+	(cd tests && ../env/bin/pytest --cov=$(dir) --cov-report=term-missing)
+
+clean:
+	rm -rf env
+	rm -f params.json

_delphi_utils_python/data_proc/geomap/geo_data_proc.py

@@ -34,6 +34,7 @@
 FIPS_MSA_OUT_FILENAME = "fips_msa_table.csv"
 FIPS_HRR_OUT_FILENAME = "fips_hrr_table.csv"
 FIPS_ZIP_OUT_FILENAME = "fips_zip_table.csv"
+FIPS_HHS_FILENAME = "fips_hhs_table.csv"
 FIPS_POPULATION_OUT_FILENAME = "fips_pop.csv"
 
 ZIP_HSA_OUT_FILENAME = "zip_hsa_table.csv"
@@ -42,8 +43,9 @@
 ZIP_MSA_OUT_FILENAME = "zip_msa_table.csv"
 ZIP_POPULATION_OUT_FILENAME = "zip_pop.csv"
 ZIP_STATE_CODE_OUT_FILENAME = "zip_state_code_table.csv"
+ZIP_HHS_FILENAME = "zip_hhs_table.csv"
 STATE_OUT_FILENAME = "state_codes_table.csv"
-STATE_HHS_OUT_FILENAME = "state_code_hhs_region_number_table.csv"
+STATE_HHS_OUT_FILENAME = "state_code_hhs_table.csv"
 JHU_FIPS_OUT_FILENAME = "jhu_uid_fips_table.csv"
 
 
@@ -217,6 +219,12 @@ def create_jhu_uid_fips_crosswalk():
             {"jhu_uid": "63072999", "fips": "72000", "weight": 1.0},
         ]
     )
+    cruise_ships = pd.DataFrame(
+        [
+            {"jhu_uid": "84088888", "fips": "88888", "weight": 1.0},
+            {"jhu_uid": "84099999", "fips": "99999", "weight": 1.0},
+        ]
+    )
 
     jhu_df = (
         pd.read_csv(JHU_FIPS_URL, dtype={"UID": str, "FIPS": str})
@@ -234,7 +242,7 @@ def create_jhu_uid_fips_crosswalk():
     # Drop the JHU UIDs that were hand-modified
     dup_ind = jhu_df["jhu_uid"].isin(
         pd.concat(
-            [hand_additions, unassigned_states, out_of_state, puerto_rico_unassigned]
+            [hand_additions, unassigned_states, out_of_state, puerto_rico_unassigned, cruise_ships]
         )["jhu_uid"].values
     )
     jhu_df.drop(jhu_df.index[dup_ind], inplace=True)
@@ -326,12 +334,12 @@ def create_state_hhs_crosswalk():
     hhs_state_pairs.append((9, "Northern Mariana Islands"))
 
     # Make dataframe
-    hhs_df = pd.DataFrame(hhs_state_pairs, columns=["hhs_region_number", "state_name"])
-    hhs_df["hhs_region_number"] = hhs_df["hhs_region_number"].astype(str)
+    hhs_df = pd.DataFrame(hhs_state_pairs, columns=["hhs", "state_name"])
+    hhs_df["hhs"] = hhs_df["hhs"].astype(str)
 
     (
         ss_df.merge(hhs_df, on="state_name", how="left")
-        .dropna()[["state_code", "hhs_region_number"]]
+        .dropna()[["state_code", "hhs"]]
         .to_csv(join(OUTPUT_DIR, STATE_HHS_OUT_FILENAME), index=False)
     )
 
@@ -391,6 +399,7 @@ def create_fips_population_table():
     df_pr = df_pr.groupby("fips").sum().reset_index()
     df_pr = df_pr[~df_pr["fips"].isin(census_pop["fips"])]
     census_pop_pr = pd.concat([census_pop, df_pr])
+
     census_pop_pr.to_csv(join(OUTPUT_DIR, FIPS_POPULATION_OUT_FILENAME), index=False)
 
 
@@ -524,6 +533,56 @@ def derive_zip_to_state_code():
     )
 
 
+def derive_fips_hhs_crosswalk():
+    """
+    Builds a crosswalk between FIPS county codes and HHS regions.
+    """
+    if not isfile(join(OUTPUT_DIR, STATE_HHS_OUT_FILENAME)):
+        create_state_hhs_crosswalk()
+    if not isfile(join(OUTPUT_DIR, FIPS_POPULATION_OUT_FILENAME)):
+        create_fips_population_table()
+
+    fips_pop = pd.read_csv(
+        join(OUTPUT_DIR, FIPS_POPULATION_OUT_FILENAME), dtype={"fips": str, "pop": int}
+    )
+    state_hhs = pd.read_csv(
+        join(OUTPUT_DIR, STATE_HHS_OUT_FILENAME),
+        dtype={"state_code": str, "state_id": str, "state_name": str},
+    )
+
+    fips_pop["state_code"] = fips_pop["fips"].str[:2]
+    (
+        fips_pop.merge(state_hhs, on="state_code", how="left")
+        .drop(columns=["state_code", "pop"])
+        .to_csv(join(OUTPUT_DIR, FIPS_HHS_FILENAME), index=False)
+    )
+
+
+def derive_zip_hhs_crosswalk():
+    """
+    Builds a crosswalk between zip code and HHS regions.
+    """
+    if not isfile(join(OUTPUT_DIR, STATE_HHS_OUT_FILENAME)):
+        create_state_hhs_crosswalk()
+    if not isfile(join(OUTPUT_DIR, ZIP_STATE_CODE_OUT_FILENAME)):
+        derive_zip_to_state_code()
+
+    zip_state = pd.read_csv(
+        join(OUTPUT_DIR, ZIP_STATE_CODE_OUT_FILENAME),
+        dtype={"zip": str, "pop": int, "state_code": str}
+    )
+    state_hhs = pd.read_csv(
+        join(OUTPUT_DIR, STATE_HHS_OUT_FILENAME),
+        dtype={"state_code": str, "state_id": str, "state_name": str},
+    )
+
+    (
+        zip_state.merge(state_hhs, on="state_code", how="left")
+        .drop(columns=["state_code", "state_id", "state_name"])
+        .to_csv(join(OUTPUT_DIR, ZIP_HHS_FILENAME), index=False)
+    )
+
+
 if __name__ == "__main__":
     create_fips_zip_crosswalk()
     create_zip_hsa_hrr_crosswalk()
@@ -538,3 +597,5 @@ def derive_zip_to_state_code():
     derive_zip_to_state_code()
     derive_fips_state_crosswalk()
     derive_zip_population_table()
+    derive_fips_hhs_crosswalk()
+    derive_zip_hhs_crosswalk()