Merge pull request #1264 from cmu-delphi/nchs-export

Refactor NCHS mortality to use delphi export util

Author: krivard

_delphi_utils_python/delphi_utils/export.py

@@ -4,6 +4,7 @@
 from os.path import join
 from typing import Optional
 
+from epiweeks import Week
 import numpy as np
 import pandas as pd
 
@@ -16,7 +17,8 @@ def create_export_csv(
     start_date: Optional[datetime] = None,
     end_date: Optional[datetime] = None,
     remove_null_samples: Optional[bool] = False,
-    write_empty_days: Optional[bool] = False
+    write_empty_days: Optional[bool] = False,
+    weekly_dates = False,
 ):
     """Export data in the format expected by the Delphi API.
 
@@ -65,10 +67,15 @@ def create_export_csv(
         dates = pd.date_range(start_date, end_date)
 
     for date in dates:
+        if weekly_dates:
+            t = Week.fromdate(pd.to_datetime(str(date)))
+            date_str = "weekly_" + str(t.year) + str(t.week).zfill(2)
+        else:
+            date_str = date.strftime('%Y%m%d')
         if metric is None:
-            export_filename = f"{date.strftime('%Y%m%d')}_{geo_res}_{sensor}.csv"
+            export_filename = f"{date_str}_{geo_res}_{sensor}.csv"
         else:
-            export_filename = f"{date.strftime('%Y%m%d')}_{geo_res}_{metric}_{sensor}.csv"
+            export_filename = f"{date_str}_{geo_res}_{metric}_{sensor}.csv"
         export_file = join(export_dir, export_filename)
         export_df = df[df["timestamp"] == date][["geo_id", "val", "se", "sample_size",]]
         if remove_null_samples:

_delphi_utils_python/setup.py

@@ -7,6 +7,7 @@
 required = [
     "boto3",
     "covidcast",
+    "epiweeks",
     "freezegun",
     "gitpython",
     "mock",

nchs_mortality/delphi_nchs_mortality/export.py

@@ -9,12 +9,11 @@
 from typing import Dict, Any
 
 import numpy as np
-from delphi_utils import S3ArchiveDiffer, get_structured_logger
+from delphi_utils import S3ArchiveDiffer, get_structured_logger, create_export_csv
 
 from .archive_diffs import arch_diffs
 from .constants import (METRICS, SENSOR_NAME_MAP,
                         SENSORS, INCIDENCE_BASE, GEO_RES)
-from .export import export_csv
 from .pull import pull_nchs_mortality_data
 
 
@@ -70,12 +69,13 @@ def run_module(params: Dict[str, Any]):
             df["sample_size"] = np.nan
             df = df[~df["val"].isnull()]
             sensor_name = "_".join([SENSOR_NAME_MAP[metric]])
-            dates = export_csv(
+            dates = create_export_csv(
                 df,
-                geo_name=GEO_RES,
+                geo_res=GEO_RES,
                 export_dir=daily_export_dir,
                 start_date=datetime.strptime(export_start_date, "%Y-%m-%d"),
                 sensor=sensor_name,
+                weekly_dates=True
             )
             if len(dates) > 0:
                 stats.append((max(dates), len(dates)))
@@ -93,12 +93,13 @@ def run_module(params: Dict[str, Any]):
                 df["sample_size"] = np.nan
                 df = df[~df["val"].isnull()]
                 sensor_name = "_".join([SENSOR_NAME_MAP[metric], sensor])
-                dates = export_csv(
+                dates = create_export_csv(
                     df,
-                    geo_name=GEO_RES,
+                    geo_res=GEO_RES,
                     export_dir=daily_export_dir,
                     start_date=datetime.strptime(export_start_date, "%Y-%m-%d"),
                     sensor=sensor_name,
+                    weekly_dates=True
                 )
                 if len(dates) > 0:
                     stats.append((max(dates), len(dates)))