Update utilities for NAN codes:

* update export utility to export the missing columns and add a test
* add a nancodes utility, which is just a dict of code constants

Author: dshemetov

_delphi_utils_python/delphi_utils/__init__.py

@@ -12,5 +12,6 @@
 from .geomap import GeoMapper
 from .smooth import Smoother
 from .signal import add_prefix
+from .nancodes import NAN_CODES
 
 __version__ = "0.1.0"

_delphi_utils_python/delphi_utils/export.py

@@ -64,7 +64,16 @@ def create_export_csv(
         else:
             export_filename = f"{date.strftime('%Y%m%d')}_{geo_res}_{metric}_{sensor}.csv"
         export_file = join(export_dir, export_filename)
-        export_df = df[df["timestamp"] == date][["geo_id", "val", "se", "sample_size",]]
+        EXPECTED_COLUMNS = [
+            "geo_id", 
+            "val", 
+            "se", 
+            "sample_size", 
+            "missing_val", 
+            "missing_se", 
+            "missing_sample_size"
+        ]
+        export_df = df[df["timestamp"] == date].filter(items=EXPECTED_COLUMNS)
         if remove_null_samples:
             export_df = export_df[export_df["sample_size"].notnull()]
         export_df = export_df.round({"val": 7, "se": 7})

_delphi_utils_python/delphi_utils/nancodes.py

@@ -0,0 +1,8 @@
+
+NAN_CODES = {
+    "Not Missing": 0,
+    "Not Applicable": 1,
+    "Region Exception": 2,
+    "Data Insufficient": 3,
+    "Unknown": 4
+}

_delphi_utils_python/tests/test_export.py

@@ -4,6 +4,7 @@
 from os.path import join
 
 import pandas as pd
+import numpy as np
 from delphi_utils import create_export_csv
 
 def _clean_directory(directory):
@@ -43,6 +44,20 @@ class TestExport:
         }
     )
 
+    # A sample data frame with missingness.
+    DF2 = pd.DataFrame(
+        {
+            "geo_id": ["51093", "51175", "51175", "51620"],
+            "timestamp": TIMES,
+            "val": [3.12345678910, np.nan, 2.2, 2.6],
+            "se": [0.15, 0.22, np.nan, 0.34],
+            "sample_size": [100, 100, 101, None],
+            "missing_val": [0, 1, 0, 0],
+            "missing_se": [0, 0, 1, 0],
+            "missing_sample_size": [0, 0, 0, 1]
+        }
+    )
+
     # Directory in which to store tests.
     TEST_DIR = "test_dir"
 
@@ -235,3 +250,23 @@ def test_export_without_null_removal(self):
             ]
         )
         assert pd.read_csv(join(self.TEST_DIR, "20200606_state_test.csv")).size > 0
+
+    def test_export_df_with_missingness(self):
+        _clean_directory(self.TEST_DIR)
+
+        create_export_csv(
+            df=self.DF2.copy(),
+            export_dir=self.TEST_DIR,
+            geo_res="state",
+            sensor="test",
+            remove_null_samples=False
+        )
+        assert _non_ignored_files_set(self.TEST_DIR) == set(
+            [
+                "20200215_state_test.csv",
+                "20200301_state_test.csv",
+                "20200315_state_test.csv",
+            ]
+        )
+        breakpoint()
+        assert pd.read_csv(join(self.TEST_DIR, "20200315_state_test.csv")).size > 0