NANs for CAN:

* add missing columns

Author: dshemetov

covid_act_now/delphi_covid_act_now/run.py

@@ -6,16 +6,38 @@
 """
 
 import numpy as np
+import pandas as pd
 
 from delphi_utils import (
     create_export_csv,
     S3ArchiveDiffer,
+    Nans
 )
 
 from .constants import GEO_RESOLUTIONS, SIGNALS
 from .geo import geo_map
 from .pull import load_data, extract_testing_metrics
 
+def add_nancodes(df, signal):
+    """Add nancodes to the dataframe."""
+    # Default missingness codes
+    df["missing_val"] = Nans.NOT_MISSING
+    df["missing_se"] = Nans.NOT_MISSING if signal == "pcr_tests_positive" else Nans.NOT_APPLICABLE
+    df["missing_sample_size"] = (
+        Nans.NOT_MISSING if signal == "pcr_tests_positive" else Nans.NOT_APPLICABLE
+    )
+
+    # Mark any nans with unknown
+    val_nans_mask = df["val"].isnull()
+    df.loc[val_nans_mask, "missing_val"] = Nans.UNKNOWN
+    if signal == "pcr_tests_positive":
+        se_nans_mask = df["se"].isnull()
+        df.loc[se_nans_mask, "missing_se"] = Nans.UNKNOWN
+        sample_size_nans_mask = df["sample_size"].isnull()
+        df.loc[sample_size_nans_mask, "missing_sample_size"] = Nans.UNKNOWN
+
+    return df
+
 def run_module(params):
     """
     Run the CAN testing metrics indicator.
@@ -56,9 +78,12 @@ def run_module(params):
     # Perform geo aggregations and export to receiving
     for geo_res in GEO_RESOLUTIONS:
         print(f"Processing {geo_res}")
+        # breakpoint()
         df = geo_map(df_county_testing, geo_res)
+        df = add_missing_current_day(df)
 
         # Export 'pcr_specimen_positivity_rate'
+        df = add_nancodes(df, "pcr_tests_positive")
         exported_csv_dates = create_export_csv(
             df,
             export_dir=export_dir,
@@ -69,6 +94,7 @@ def run_module(params):
         df["val"] = df["sample_size"]
         df["sample_size"] = np.nan
         df["se"] = np.nan
+        df = add_nancodes(df, "pcr_tests_total")
         exported_csv_dates = create_export_csv(
             df,
             export_dir=export_dir,