Merge pull request #1545 from cmu-delphi/release/indicators_v0.3.6_utils_v0.3.2

Release covidcast-indicators 0.3.6

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.5
+current_version = 0.3.6
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.1
+current_version = 0.3.2
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.3.1"
+__version__ = "0.3.2"

_delphi_utils_python/delphi_utils/archive.py

@@ -49,6 +49,10 @@
 Files = List[str]
 FileDiffMap = Dict[str, Optional[str]]
 
+EXPORT_CSV_DTYPES = {
+    "geo_id": str, "val": float, "se": float, "sample_size": float,
+    "missing_val": "Int64", "missing_se": "Int64", "missing_sample_size": "Int64"
+}
 
 def diff_export_csv(
     before_csv: str,
@@ -75,15 +79,10 @@ def diff_export_csv(
         changed_df is the pd.DataFrame of common rows from after_csv with changed values.
         added_df is the pd.DataFrame of added rows from after_csv.
     """
-    export_csv_dtypes = {
-        "geo_id": str, "val": float, "se": float, "sample_size": float,
-        "missing_val": int, "missing_se": int, "missing_sample_size": int
-    }
-
-    before_df = pd.read_csv(before_csv, dtype=export_csv_dtypes)
+    before_df = pd.read_csv(before_csv, dtype=EXPORT_CSV_DTYPES)
     before_df.set_index("geo_id", inplace=True)
     before_df = before_df.round({"val": 7, "se": 7})
-    after_df = pd.read_csv(after_csv, dtype=export_csv_dtypes)
+    after_df = pd.read_csv(after_csv, dtype=EXPORT_CSV_DTYPES)
     after_df.set_index("geo_id", inplace=True)
     after_df = after_df.round({"val": 7, "se": 7})
     deleted_idx = before_df.index.difference(after_df.index)
@@ -93,20 +92,21 @@ def diff_export_csv(
     before_df_cmn = before_df.reindex(common_idx)
     after_df_cmn = after_df.reindex(common_idx)
 
-    # If CSVs have different columns (no missingness), mark all values as new
-    if ("missing_val" in before_df_cmn.columns) ^ ("missing_val" in after_df_cmn.columns):
+    # If new CSV has missingness columns, but old doesn't, mark all values as new
+    if ("missing_val" not in before_df_cmn.columns) & ("missing_val" in after_df_cmn.columns):
         same_mask = after_df_cmn.copy()
         same_mask.loc[:] = False
     else:
         # Exact comparisons, treating NA == NA as True
         same_mask = before_df_cmn == after_df_cmn
         same_mask |= pd.isna(before_df_cmn) & pd.isna(after_df_cmn)
 
-    # Code deleted entries as nans with the deleted missing code
+    # Any deleted entries become rows with nans and the deleted missing code
     deleted_df = before_df.loc[deleted_idx, :].copy()
     deleted_df[["val", "se", "sample_size"]] = np.nan
-    if "missing_val" in after_df_cmn.columns:
-        deleted_df[["missing_val", "missing_se", "missing_sample_size"]] = Nans.DELETED
+    # If the new file doesn't have missing columsn, then when the deleted, changed, and added
+    # rows are concatenated (in diff_exports), they will default to NA
+    deleted_df[["missing_val", "missing_se", "missing_sample_size"]] = Nans.DELETED
 
     return (
         deleted_df,

_delphi_utils_python/setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.3.1",
+    version="0.3.2",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",