Merge branch 'main-nan-testing' into nans_nchs

Author: dshemetov

Jenkinsfile

@@ -19,7 +19,10 @@ pipeline {
     stages {
         stage('Build and Package') {
             when {
+                anyOf {
                 branch "main";
+                branch "main-nan-testing";
+                }
             }
             steps {
                 script {
@@ -34,7 +37,10 @@ pipeline {
         }
         stage('Deploy staging') {
             when {
+                anyOf {
                 branch "main";
+                branch "main-nan-testing";
+                }
             }
             steps {
                 script {

_delphi_utils_python/DEVELOP.md

@@ -0,0 +1,56 @@
+# DELPHI Common Utility Functions (Python)
+
+This directory contains the Python module `delphi_utils`. It includes a number of
+common functions that are useful across multiple indicators.
+
+## Installing the Module
+
+To install the module in your default version of Python, run the
+following from this directory:
+
+```
+pip install .
+```
+
+As described in each of the indicator code directories, you will want to install
+this module within a virtual environment when testing the various code bases.
+
+### Testing the code
+
+To do a static test of the code style, it is recommended to run **pylint** on
+the module. To do this, run the following from the main module directory:
+
+```
+pylint delphi_utils
+```
+
+The most aggressive checks are turned off; only relatively important issues
+should be raised and they should be manually checked (or better, fixed).
+
+Unit tests are also included in the module. These should be run by first
+installing the module into a virtual environment:
+
+```
+python -m venv env
+source env/bin/activate
+pip install .
+```
+
+And then running the unit tests with:
+
+```
+(cd tests && ../env/bin/pytest --cov=delphi_utils --cov-report=term-missing)
+```
+
+The output will show the number of unit tests that passed and failed, along
+with the percentage of code covered by the tests. None of the tests should
+fail and the code lines that are not covered by unit tests should be small and
+should not include critical sub-routines.
+
+When you are finished, the virtual environment can be deactivated and
+(optionally) removed.
+
+```
+deactivate
+rm -r env
+```

_delphi_utils_python/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 The Delphi Group at Carnegie Mellon University
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

_delphi_utils_python/README.md

@@ -1,56 +1,21 @@
-# DELPHI Common Utility Functions (Python)
-
-This director contains the Python module `delphi_utils`. It includes a number of
-common functions that are useful across multiple indicators.
-
-## Installing the Module
-
-To install the module in your default version of Python, run the
-following from this directory:
-
-```
-pip install .
-```
-
-As described in each of the indicator code directories, you will want to install
-this module within a virtual environment when testing the various code bases.
-
-### Testing the code
-
-To do a static test of the code style, it is recommended to run **pylint** on
-the module. To do this, run the following from the main module directory:
-
-```
-pylint delphi_utils
-```
-
-The most aggressive checks are turned off; only relatively important issues
-should be raised and they should be manually checked (or better, fixed).
-
-Unit tests are also included in the module. These should be run by first
-installing the module into a virtual environment:
-
-```
-python -m venv env
-source env/bin/activate
-pip install .
-```
-
-And then running the unit tests with:
-
-```
-(cd tests && ../env/bin/pytest --cov=delphi_utils --cov-report=term-missing)
-```
-
-The output will show the number of unit tests that passed and failed, along
-with the percentage of code covered by the tests. None of the tests should
-fail and the code lines that are not covered by unit tests should be small and
-should not include critical sub-routines.
-
-When you are finished, the virtual environment can be deactivated and
-(optionally) removed.
-
-```
-deactivate
-rm -r env
-```
+# Delphi Python Utilities
+
+This package provides various utilities used by the [Delphi group](https://delphi.cmu.edu/) at [Carnegie Mellon
+University](https://www.cmu.edu) for its data pipelines and analyses.
+
+Submodules:
+- `archive`: Diffing and archiving CSV files.
+- `export`: DataFrame to CSV export.
+- `geomap`: Mappings between geographic resolutions.
+- `logger`: Structured JSON logger.
+- `nancodes`: Enum constants encoding not-a-number cases.
+- `runner`: Orchestrator for running an indicator pipeline.
+- `signal`: Indicator (signal) naming.
+- `slack_notifier`:  Slack notification integration.
+- `smooth`: Data smoothing functions.
+- `utils`: JSON parameter interactions.
+- `validator`: Data sanity checks and anomaly detection.
+
+
+Source code can be found here: 
+[https://github.com/cmu-delphi/covidcast-indicators/](https://github.com/cmu-delphi/covidcast-indicators/)

_delphi_utils_python/delphi_utils/nancodes.py

@@ -1,10 +1,4 @@
-"""Provides unified not-a-number codes for the indicators.
-
-Currently requires a manual sync between the covidcast-indicators
-and the delphi-epidata repo.
-* in covidcast-indicators: _delphi_utils_python/delphi_utils
-* in delphi-epidata: src/acquisition/covidcast
-"""
+"""Unified not-a-number codes for CMU Delphi codebase."""
 
 from enum import IntEnum
 

_delphi_utils_python/delphi_utils/validator/scripts/unique_geoids.R

@@ -1,11 +1,15 @@
 from setuptools import setup
 from setuptools import find_packages
 
+with open("README.md", "r") as f:
+  long_description = f.read()
+
 required = [
     "boto3",
     "covidcast",
     "freezegun",
     "gitpython",
+    "mock",
     "moto",
     "numpy",
     "pandas>=1.1.0",
@@ -22,6 +26,8 @@
     name="delphi_utils",
     version="0.1.0",
     description="Shared Utility Functions for Indicators",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
     author="",
     author_email="",
     url="https://github.com/cmu-delphi/",

_delphi_utils_python/setup.py

@@ -63,6 +63,7 @@ importFrom(dplyr,across)
 importFrom(dplyr,all_of)
 importFrom(dplyr,anti_join)
 importFrom(dplyr,arrange)
+importFrom(dplyr,bind_cols)
 importFrom(dplyr,bind_rows)
 importFrom(dplyr,case_when)
 importFrom(dplyr,coalesce)

facebook/delphiFacebook/NAMESPACE

@@ -264,15 +264,10 @@ summarize_aggs <- function(df, crosswalk_data, aggregations, geo_level, params)
   }
 
   ## Find all unique groups and associated frequencies, saved in column `Freq`.
-  # Keep rows with missing values initially so that we get the correct column
-  # names. Explicitly drop groups with missing values in second step.
   unique_groups_counts <- as.data.frame(
     table(df[, group_vars, with=FALSE], exclude=NULL, dnn=group_vars), 
     stringsAsFactors=FALSE
   )
-  unique_groups_counts <- unique_groups_counts[
-    complete.cases(unique_groups_counts[, group_vars]),
-  ]
 
   # Drop groups with less than threshold sample size.
   unique_groups_counts <- filter(unique_groups_counts, Freq >= params$num_filter)
@@ -327,9 +322,10 @@ summarize_aggs <- function(df, crosswalk_data, aggregations, geo_level, params)
     aggregation <- aggregations$id[row]
     group_vars <- aggregations$group_by[[row]]
     post_fn <- aggregations$post_fn[[row]]
-
+    
+    # Keep only aggregations where the main value, `val`, is present.
     dfs_out[[aggregation]] <- dfs_out[[aggregation]][
-      rowSums(is.na(dfs_out[[aggregation]][, c("val", "sample_size", group_vars)])) == 0,
+      rowSums(is.na(dfs_out[[aggregation]][, c("val", "sample_size")])) == 0,
     ]
 
     dfs_out[[aggregation]] <- apply_privacy_censoring(dfs_out[[aggregation]], params)

facebook/delphiFacebook/R/contingency_aggregate.R

@@ -13,12 +13,11 @@
 #'   be a non-integer
 #' 
 #' @return a list of named means and other descriptive statistics
-compute_numeric <- function(response, weight, sample_size, total_represented)
+compute_household_binary <- function(response, weight, sample_size, total_represented)
 {
   response_mean <- compute_count_response(response, weight, sample_size)
   response_mean$sample_size <- sample_size
   response_mean$represented <- total_represented
-  response_mean$se <- NA_real_
 
   return(response_mean)
 }