Merge pull request #605 from cmu-delphi/main

Release nchs_mortality to production

Author: korlaxxalrok

Jenkinsfile

@@ -45,6 +45,7 @@ pipeline {
                     }
                     parallel deploy_staging
                 }
+                sh "jenkins/deploy-staging-api-match-list.sh"
             }
         }
         stage('Deploy production') {

_delphi_utils_python/Makefile

@@ -12,7 +12,8 @@ install: venv
 
 lint:
 	. env/bin/activate; \
-	pylint $(dir)
+	pylint $(dir); \
+	pydocstyle $(dir)
 
 test:
 	. env/bin/activate ;\

_delphi_utils_python/delphi_utils/__init__.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
-"""Common Utility Functions to Support DELPHI Indicators
-"""
+"""Common Utility Functions to Support DELPHI Indicators."""
 
 from __future__ import absolute_import
 

_delphi_utils_python/delphi_utils/archive.py

@@ -1,5 +1,6 @@
 """
 Utilities for diffing and archiving covidcast export CSVs.
+
 Aims to simplify the creation of issues for new and backfilled value for indicators.
 Also handles archiving of export CSVs to some backend (git, S3 etc.) before replacing them.
 
@@ -52,6 +53,7 @@ def diff_export_csv(
 ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     """
     Find differences in exported covidcast CSVs, using geo_id as the index.
+
     Treats NA == NA as True.
 
     Parameters
@@ -68,7 +70,6 @@ def diff_export_csv(
         changed_df is the pd.DataFrame of common rows from after_csv with changed values.
         added_df is the pd.DataFrame of added rows from after_csv.
     """
-
     export_csv_dtypes = {"geo_id": str, "val": float,
                          "se": float, "sample_size": float}
 
@@ -99,7 +100,7 @@ def run_module(archive_type: str,
                cache_dir: str,
                export_dir: str,
                **kwargs):
-    """Builds and runs an ArchiveDiffer.
+    """Build and run an ArchiveDiffer.
 
     Parameters
     ----------
@@ -132,13 +133,11 @@ def run_module(archive_type: str,
 
 
 class ArchiveDiffer:
-    """
-    Base class for performing diffing and archiving of exported covidcast CSVs
-    """
+    """Base class for performing diffing and archiving of exported covidcast CSVs."""
 
     def __init__(self, cache_dir: str, export_dir: str):
         """
-        Initialize an ArchiveDiffer
+        Initialize an ArchiveDiffer.
 
         Parameters
         ----------
@@ -157,15 +156,17 @@ def __init__(self, cache_dir: str, export_dir: str):
 
     def update_cache(self):
         """
-        For making sure cache_dir is updated correctly from a backend.
+        Make sure cache_dir is updated correctly from a backend.
+
         To be implemented by specific archiving backends.
         Should set self._cache_updated = True after verifying cache is updated.
         """
         raise NotImplementedError
 
     def diff_exports(self) -> Tuple[Files, FileDiffMap, Files]:
         """
-        Finds diffs across and within CSV files, from cache_dir to export_dir.
+        Find diffs across and within CSV files, from cache_dir to export_dir.
+
         Should be called after update_cache() succeeds. Only works on *.csv files,
         ignores every other file.
 
@@ -223,7 +224,8 @@ def diff_exports(self) -> Tuple[Files, FileDiffMap, Files]:
 
     def archive_exports(self, exported_files: Files) -> Tuple[Files, Files]:
         """
-        Handles actual archiving of files, depending on specific backend.
+        Handle actual archiving of files, depending on specific backend.
+
         To be implemented by specific archiving backends.
 
         Parameters
@@ -241,6 +243,8 @@ def archive_exports(self, exported_files: Files) -> Tuple[Files, Files]:
 
     def filter_exports(self, common_diffs: FileDiffMap):
         """
+        Filter export directory to only contain relevant files.
+
         Filters down the export_dir to only contain:
         1) New files, 2) Changed files, filtered-down to the ADDED and CHANGED rows only.
         Should be called after archive_exports() so we archive the raw exports before
@@ -269,7 +273,7 @@ def filter_exports(self, common_diffs: FileDiffMap):
                 replace(diff_file, exported_file)
 
     def run(self):
-        """Runs the differ and archives the changed and new files."""
+        """Run the differ and archive the changed and new files."""
         self.update_cache()
 
         # Diff exports, and make incremental versions
@@ -293,7 +297,8 @@ def run(self):
 
 class S3ArchiveDiffer(ArchiveDiffer):
     """
-    AWS S3 backend for archving
+    AWS S3 backend for archiving.
+
     Archives CSV files into a S3 bucket, with keys "{indicator_prefix}/{csv_file_name}".
     Ideally, versioning should be enabled in this bucket to track versions of each CSV file.
     """
@@ -306,6 +311,7 @@ def __init__(
     ):
         """
         Initialize a S3ArchiveDiffer.
+
         See this link for possible aws_credentials kwargs:
         https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session
 
@@ -330,9 +336,7 @@ def __init__(
         self.indicator_prefix = indicator_prefix
 
     def update_cache(self):
-        """
-        For making sure cache_dir is updated with all latest files from the S3 bucket.
-        """
+        """Make sure cache_dir is updated with all latest files from the S3 bucket."""
         # List all indicator-related objects from S3
         archive_objects = self.bucket.objects.filter(
             Prefix=self.indicator_prefix).all()
@@ -358,7 +362,7 @@ def archive_exports(self,  # pylint: disable=arguments-differ
         update_s3: bool = True
     ) -> Tuple[Files, Files]:
         """
-        Handles actual archiving of files to the S3 bucket.
+        Handle actual archiving of files to the S3 bucket.
 
         Parameters
         ----------
@@ -398,7 +402,8 @@ def archive_exports(self,  # pylint: disable=arguments-differ
 
 class GitArchiveDiffer(ArchiveDiffer):
     """
-    Local git repo backend for archiving
+    Local git repo backend for archiving.
+
     Archives CSV files into a local git repo as commits.
     Assumes that a git repository is already set up.
     """
@@ -446,7 +451,8 @@ def __init__(
 
     def get_branch(self, branch_name: Optional[str] = None) -> Head:
         """
-        Retrieves a Head object representing a branch of specified name.
+        Retrieve a Head object representing a branch of specified name.
+
         Creates the branch from the current active branch if does not exist yet.
 
         Parameters
@@ -469,6 +475,8 @@ def get_branch(self, branch_name: Optional[str] = None) -> Head:
     @contextmanager
     def archiving_branch(self):
         """
+        Context manager for checking out a branch.
+
         Useful for checking out self.branch within a context, then switching back
         to original branch when finished.
         """
@@ -482,8 +490,9 @@ def archiving_branch(self):
 
     def update_cache(self):
         """
+        Check if cache_dir is clean: has everything nicely committed if override_dirty=False.
+
         Since we are using a local git repo, assumes there is nothing to update from.
-        Checks if cache_dir is clean: has everything nice committed if override_dirty=False
         """
         # Make sure cache directory is clean: has everything nicely committed
         if not self.override_dirty:
@@ -495,14 +504,16 @@ def update_cache(self):
 
     def diff_exports(self) -> Tuple[Files, FileDiffMap, Files]:
         """
-        Same as base class diff_exports, but in context of specified branch
+        Find diffs across and within CSV files, from cache_dir to export_dir.
+
+        Same as base class diff_exports, but in context of specified branch.
         """
         with self.archiving_branch():
             return super().diff_exports()
 
     def archive_exports(self, exported_files: Files) -> Tuple[Files, Files]:
         """
-        Handles actual archiving of files to the local git repo.
+        Handle actual archiving of files to the local git repo.
 
         Parameters
         ----------

_delphi_utils_python/delphi_utils/geomap.py

@@ -91,8 +91,9 @@ class GeoMapper:  # pylint: disable=too-many-public-methods
     """
 
     def __init__(self):
-        """Initialize geomapper. Holds loading the crosswalk tables
-        until a conversion function is first used.
+        """Initialize geomapper.
+
+        Holds loading the crosswalk tables until a conversion function is first used.
 
         Parameters
         ---------
@@ -110,7 +111,7 @@ def __init__(self):
 
     # Utility functions
     def _load_crosswalk(self, from_code, to_code):
-        """Loads the crosswalk from from_code -> to_code."""
+        """Load the crosswalk from from_code -> to_code."""
         stream = pkg_resources.resource_stream(
             __name__, self.crosswalk_filepaths[from_code][to_code]
         )
@@ -189,7 +190,7 @@ def _load_crosswalk(self, from_code, to_code):
 
     @staticmethod
     def convert_fips_to_mega(data, fips_col="fips", mega_col="megafips"):
-        """convert fips string to a megafips string"""
+        """Convert fips string to a megafips string."""
         data = data.copy()
         data[mega_col] = data[fips_col].astype(str).str.zfill(5)
         data[mega_col] = data[mega_col].str.slice_replace(start=2, stop=5, repl="000")
@@ -205,7 +206,7 @@ def megacounty_creation(
         date_col="date",
         mega_col="megafips",
     ):
-        """create megacounty column
+        """Create megacounty column.
 
         Parameters
         ---------
@@ -412,8 +413,9 @@ def replace_geocode(
 
     def add_population_column(self, data, geocode_type, geocode_col=None, dropna=True):
         """
-        Appends a population column to a dataframe, based on the FIPS or ZIP code. If no
-        dataframe is provided, the full crosswalk from geocode to population is returned.
+        Append a population column to a dataframe, based on the FIPS or ZIP code.
+
+        If no dataframe is provided, the full crosswalk from geocode to population is returned.
 
         Parameters
         ---------
@@ -464,7 +466,7 @@ def fips_to_megacounty(
         mega_col="megafips",
         count_cols=None,
     ):
-        """Convert and aggregate from FIPS to megaFIPS
+        """Convert and aggregate from FIPS to megaFIPS.
 
         Parameters
         ---------

_delphi_utils_python/delphi_utils/signal.py

@@ -2,7 +2,8 @@
 import covidcast
 
 def add_prefix(signal_names, wip_signal, prefix="wip_"):
-    """Adds prefix to signal if there is a WIP signal
+    """Add prefix to signal if there is a WIP signal.
+
     Parameters
     ----------
     signal_names: List[str]
@@ -18,7 +19,6 @@ def add_prefix(signal_names, wip_signal, prefix="wip_"):
     List of signal names
         wip/non wip signals for further computation
     """
-
     if wip_signal is True:
         return [prefix + signal for signal in signal_names]
     if isinstance(wip_signal, list):
@@ -37,7 +37,8 @@ def add_prefix(signal_names, wip_signal, prefix="wip_"):
 
 
 def public_signal(signal):
-    """Checks if the signal name is already public using COVIDcast
+    """Check if the signal name is already public using COVIDcast.
+
     Parameters
     ----------
     signal : str

_delphi_utils_python/delphi_utils/utils.py

@@ -5,7 +5,7 @@
 from shutil import copyfile
 
 def read_params():
-    """Reads a file named 'params.json' in the current working directory.
+    """Read a file named 'params.json' in the current working directory.
 
     If the file does not exist, it copies the file 'params.json.template' to
     'param.json' and then reads the file.

ansible/ansible-deploy-staging-api-proxy-match-list.yaml

@@ -0,0 +1,14 @@
+---
+- hosts: api_proxy_staging
+  remote_user: deploy
+  vars_files:
+    - vars.yaml
+    - vault.yaml
+  tasks:
+    - name: Set staging api proxy openresty signal match list template.
+      template:
+        src: "templates/staging-api-match-list.j2"
+        dest: "/common/staging-api-match-list"
+        owner: "deploy"
+        group: "deploy"
+        mode: "0777"

ansible/inventory

@@ -3,3 +3,6 @@ delphi-master-prod-01.delphi.cmu.edu
 
 [runtime_host_staging]
 app-mono-dev-01.delphi.cmu.edu
+
+[api_proxy_staging]
+api-staging.delphi.cmu.edu

ansible/templates/changehc-params-prod.json.j2

@@ -2,8 +2,14 @@
   "static_file_dir": "./static",
   "export_dir": "/common/covidcast/receiving/chng",
   "cache_dir": "./cache",
-  "input_denom_file": null,
-  "input_covid_file": null,
+  "input_files": {
+    "denom": null,
+    "covid": null,
+    "flu": null,
+    "mixed": null,
+    "flu_like": null,
+    "covid_like": null
+  },
   "start_date": "2020-02-01",
   "end_date": null,
   "drop_date": null,
@@ -13,6 +19,7 @@
   "parallel": false,
   "geos": ["state", "msa", "hrr", "county"],
   "weekday": [true, false],
+  "types": ["covid","cli"],
   "wip_signal": "",
   "aws_credentials": {
     "aws_access_key_id": "",

ansible/templates/nchs_mortality-params-prod.json.j2

@@ -0,0 +1,15 @@
+{
+  "export_start_date": "2020-02-01",
+  "static_file_dir": "./static",
+  "export_dir": "/common/covidcast/receiving/nchs-mortality",
+  "cache_dir": "./cache",
+  "daily_export_dir": "./daily_receiving",
+  "daily_cache_dir": "./daily_cache",
+  "token": "{{ nchs_mortality_token }}",
+  "mode":"",
+  "aws_credentials": {
+    "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
+    "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+  },
+  "bucket_name": "delphi-covidcast-indicator-output"
+}

ansible/templates/staging-api-match-list.j2

@@ -0,0 +1,4 @@
+data_source=quidel-staging&signal=covid_ag_
+data_source=chng
+data_source=safegraph
+data_source=google-symptoms

ansible/vars.yaml

@@ -21,3 +21,4 @@ changehc_sftp_host: "{{ vault_changehc_sftp_host }}"
 changehc_sftp_port: "{{ vault_changehc_sftp_port }}"
 changehc_sftp_user: "{{ vault_changehc_sftp_user }}"
 changehc_sftp_password: "{{ vault_changehc_sftp_password }}"
+nchs_mortality_token: "{{ vault_nchs_mortality_token }}"