using epidata metadata to determine stalenes for nhsn pulling

nhsn/delphi_nhsn/constants.py

@@ -1,5 +1,7 @@
 """Registry for signal names."""
 
+from datetime import timedelta
+
 GEOS = ["state", "nation", "hhs"]
 
 MAIN_DATASET_ID = "ua7e-t2fy"
@@ -62,3 +64,5 @@
     f"{NUM_HOSP_REPORTING_FLU}_prelim": float,
     f"{NUM_HOSP_REPORTING_RSV}_prelim": float,
 }
+
+RECENTLY_UPDATED_DIFF = timedelta(hours=36)

nhsn/delphi_nhsn/pull.py

@@ -4,7 +4,7 @@
 import logging
 import random
 import time
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
 from typing import Optional
 from urllib.error import HTTPError
@@ -13,7 +13,15 @@
 from delphi_utils import create_backup_csv
 from sodapy import Socrata
 
-from .constants import MAIN_DATASET_ID, PRELIM_DATASET_ID, PRELIM_SIGNALS_MAP, PRELIM_TYPE_DICT, SIGNALS_MAP, TYPE_DICT
+from .constants import (
+    MAIN_DATASET_ID,
+    PRELIM_DATASET_ID,
+    PRELIM_SIGNALS_MAP,
+    PRELIM_TYPE_DICT,
+    RECENTLY_UPDATED_DIFF,
+    SIGNALS_MAP,
+    TYPE_DICT,
+)
 
 
 def check_last_updated(socrata_token, dataset_id, logger):
@@ -40,7 +48,8 @@ def check_last_updated(socrata_token, dataset_id, logger):
 
         updated_timestamp = datetime.utcfromtimestamp(int(response["rowsUpdatedAt"]))
         now = datetime.utcnow()
-        recently_updated_source = (now - updated_timestamp) < timedelta(days=1)
+        # currently set to run twice a week, RECENTLY_UPDATED_DIFF may need adjusting based on the cadence
+        recently_updated_source = (now - updated_timestamp) < RECENTLY_UPDATED_DIFF
 
         prelim_prefix = "Preliminary " if dataset_id == PRELIM_DATASET_ID else ""
         if recently_updated_source:

nhsn/tests/test_pull.py

@@ -1,5 +1,6 @@
 import glob
 import time
+from datetime import datetime
 from unittest.mock import patch, MagicMock
 import os
 import pytest
@@ -12,7 +13,7 @@
     pull_data_from_file,
     check_last_updated
 )
-from delphi_nhsn.constants import TYPE_DICT, PRELIM_TYPE_DICT, PRELIM_DATASET_ID, MAIN_DATASET_ID
+from delphi_nhsn.constants import TYPE_DICT, PRELIM_TYPE_DICT, PRELIM_DATASET_ID, MAIN_DATASET_ID, RECENTLY_UPDATED_DIFF
 
 from delphi_utils import get_structured_logger
 from conftest import TEST_DATA, PRELIM_TEST_DATA, TEST_DIR
@@ -158,7 +159,7 @@ def test_pull_nhsn_data_backup(self, mock_socrata, dataset, caplog, params):
 
 
     @pytest.mark.parametrize('dataset', DATASETS, ids=["data", "prelim_data"])
-    @pytest.mark.parametrize("updatedAt", [time.time(), time.time() - 172800], ids=["updated", "stale"])
+    @pytest.mark.parametrize("updatedAt", [time.time(), time.time() - 172800, time.time() - 108000], ids=["updated", "stale", "updated_late"])
     @patch("delphi_nhsn.pull.Socrata")
     def test_check_last_updated(self, mock_socrata, dataset, updatedAt, caplog):
         mock_client = MagicMock()
@@ -169,8 +170,9 @@ def test_check_last_updated(self, mock_socrata, dataset, updatedAt, caplog):
         check_last_updated(mock_client, dataset["id"], logger)
 
         # Check that get method was called with correct arguments
-        now = time.time()
-        if now - updatedAt < 60:
+        now_datetime = datetime.utcfromtimestamp(time.time())
+        updatedAt_datetime = datetime.utcfromtimestamp(updatedAt)
+        if now_datetime - updatedAt_datetime < RECENTLY_UPDATED_DIFF:
             assert f"{dataset['msg_prefix']}NHSN data was recently updated; Pulling data" in caplog.text
         else:
             stale_msg = f"{dataset['msg_prefix']}NHSN data is stale; Skipping"