NANs quidel covidtest:

* add missing columns

Author: dshemetov

quidel_covidtest/delphi_quidel_covidtest/run.py

@@ -11,7 +11,8 @@
 from delphi_utils import (
     add_prefix,
     create_export_csv,
-    get_structured_logger
+    get_structured_logger,
+    Nans
 )
 
 from .constants import (END_FROM_TODAY_MINUS,
@@ -32,6 +33,22 @@ def log_exit(start_time, logger):
     logger.info("Completed indicator run",
                 elapsed_time_in_seconds=elapsed_time_in_seconds)
 
+def add_nancodes(df):
+    """Add nancodes to the dataframe."""
+    # Default missingness codes
+    df["missing_val"] = Nans.NOT_MISSING
+    df["missing_se"] = Nans.NOT_MISSING
+    df["missing_sample_size"] = Nans.NOT_MISSING
+
+    # Mark any remaining nans with unknown
+    remaining_nans_mask = df["val"].isnull()
+    df.loc[remaining_nans_mask, "missing_val"] = Nans.UNKNOWN
+    remaining_nans_mask = df["se"].isnull()
+    df.loc[remaining_nans_mask, "missing_se"] = Nans.UNKNOWN
+    remaining_nans_mask = df["sample_size"].isnull()
+    df.loc[remaining_nans_mask, "missing_sample_size"] = Nans.UNKNOWN
+    return df
+
 def run_module(params: Dict[str, Any]):
     """Run the quidel_covidtest indicator.
 
@@ -103,6 +120,7 @@ def run_module(params: Dict[str, Any]):
             state_groups, smooth=smoothers[sensor][1],
             device=smoothers[sensor][0], first_date=first_date,
             last_date=last_date)
+        state_df = add_nancodes(state_df)
         create_export_csv(state_df, geo_res="state", sensor=sensor, export_dir=export_dir,
                           start_date=export_start_date, end_date=export_end_date)
 
@@ -115,6 +133,7 @@ def run_module(params: Dict[str, Any]):
                 state_groups, geo_data, res_key, smooth=smoothers[sensor][1],
                 device=smoothers[sensor][0], first_date=first_date,
                 last_date=last_date)
+            res_df = add_nancodes(res_df)
             create_export_csv(res_df, geo_res=geo_res, sensor=sensor, export_dir=export_dir,
                               start_date=export_start_date, end_date=export_end_date,
                               remove_null_samples=True)

quidel_covidtest/tests/test_run.py

@@ -64,7 +64,11 @@ def test_output_files(self, clean_receiving_dir):
         df = pd.read_csv(
             join("./receiving", "20200718_state_covid_ag_smoothed_pct_positive.csv")
         )
-        assert (df.columns.values == ["geo_id", "val", "se", "sample_size"]).all()
+        expected_columns = [
+            "geo_id", "val", "se", "sample_size",
+            "missing_val", "missing_se", "missing_sample_size"
+        ]
+        assert (df.columns.values == expected_columns).all()
 
         # test_intermediate_file
         flag = None