NANS doctor visits:

* write missing columns
* update test files

Author: dshemetov

doctor_visits/delphi_doctor_visits/update_sensor.py

@@ -17,7 +17,7 @@
 import pandas as pd
 
 # first party
-from delphi_utils import Weekday
+from delphi_utils import Weekday, Nans
 from .config import Config
 from .geo_maps import GeoMaps
 from .sensor import DoctorVisitsSensor
@@ -43,7 +43,10 @@ def write_to_csv(output_df: pd.DataFrame, geo_level, se, out_name, logger, outpu
                                         out_name)
         single_date_df = output_df[output_df["date"] == d]
         with open(filename, "w") as outfile:
-            outfile.write("geo_id,val,se,direction,sample_size\n")
+            outfile.write(
+                "geo_id,val,se,direction,sample_size,"
+                "missing_val,missing_se,missing_sample_size\n"
+            )
 
             for line in single_date_df.itertuples():
                 geo_id = line.geo_id
@@ -57,11 +60,26 @@ def write_to_csv(output_df: pd.DataFrame, geo_level, se, out_name, logger, outpu
                 if se:
                     assert sensor > 0 and se_val > 0, "p=0, std_err=0 invalid"
                     outfile.write(
-                        "%s,%f,%s,%s,%s\n" % (geo_id, sensor, se_val, "NA", "NA"))
+                        "%s,%f,%s,%s,%s,%d,%d,%d\n" %
+                        (
+                            geo_id, sensor, se, "NA", "NA",
+                            Nans.NOT_MISSING.value,
+                            Nans.NOT_MISSING.value,
+                            Nans.NOT_APPLICABLE.value
+                        )
+                    )
                 else:
                     # for privacy reasons we will not report the standard error
                     outfile.write(
-                        "%s,%f,%s,%s,%s\n" % (geo_id, sensor, "NA", "NA", "NA"))
+                        "%s,%f,%s,%s,%s,%d,%d,%d\n" %
+                        (
+                            geo_id, sensor, "NA", "NA", "NA",
+                            Nans.NOT_MISSING.value,
+                            Nans.CENSORED.value,
+                            Nans.NOT_APPLICABLE.value
+                        )
+                    )
+
                 out_n += 1
     logger.debug(f"wrote {out_n} rows for {geo_level}")