2129 fix hrr for nssp

nssp/delphi_nssp/constants.py

@@ -1,5 +1,7 @@
 """Registry for variations."""
 
+DATASET_ID = "rdmq-nq56"
+
 GEOS = [
     "hrr",
     "msa",

nssp/delphi_nssp/pull.py

@@ -13,7 +13,7 @@
 from delphi_utils import create_backup_csv
 from sodapy import Socrata
 
-from .constants import NEWLINE, SIGNALS, SIGNALS_MAP, TYPE_DICT
+from .constants import DATASET_ID, NEWLINE, SIGNALS, SIGNALS_MAP, TYPE_DICT
 
 
 def print_callback(remote_file_name, logger, bytes_so_far, bytes_total, progress_chunks):
@@ -148,7 +148,7 @@ def pull_nssp_data(
         Dataframe as described above.
     """
     if not custom_run:
-        socrata_results = pull_with_socrata_api(socrata_token, "rdmq-nq56")
+        socrata_results = pull_with_socrata_api(socrata_token, DATASET_ID)
         df_ervisits = pd.DataFrame.from_records(socrata_results)
         create_backup_csv(df_ervisits, backup_dir, custom_run, logger=logger)
         logger.info("Number of records grabbed", num_records=len(df_ervisits), source="Socrata API")

nssp/delphi_nssp/run.py

@@ -118,9 +118,10 @@ def run_module(params, logger=None):
                 )
             elif geo == "hrr":
                 df = df[["fips", "val", "timestamp"]]
-                # fips -> hrr has a weighted version
-                df = geo_mapper.replace_geocode(df, "fips", "hrr")
-                df = df.rename(columns={"hrr": "geo_id"})
+                df = geo_mapper.add_population_column(df, geocode_type="fips", geocode_col="fips")
+                df = geo_mapper.add_geocode(df, "fips", "hrr", from_col="fips", new_col="geo_id")
+                df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population")
+                df = df.rename(columns={"weighted_val": "val"})
             elif geo == "msa":
                 df = df[["fips", "val", "timestamp"]]
                 # fips -> msa doesn't have a weighted version, so we need to add columns and sum ourselves

nssp/tests/conftest.py

@@ -0,0 +1,89 @@
+import copy
+import json
+import time
+from unittest.mock import patch, MagicMock
+
+import pytest
+from pathlib import Path
+
+from delphi_nssp.run import run_module
+from delphi_nssp.constants import DATASET_ID
+
+TEST_DIR = Path(__file__).parent
+
+# test data generated with following url with socrata:
+# https://data.cdc.gov/resource/rdmq-nq56.json?$where=week_end >= '2022-10-01T00:00:00.000' AND week_end <= '2022-10-20T00:00:00.000'
+
+with open(f"{TEST_DIR}/test_data/page.json", "r") as f:
+    TEST_DATA = json.load(f)
+
+with open(f"{TEST_DIR}/test_data/page_100_hrr.json", "r") as f:
+    HRR_TEST_DATA = json.load(f)
+
+@pytest.fixture(scope="session")
+def params():
+    params = {
+        "common": {
+            "export_dir": f"{TEST_DIR}/receiving",
+            "log_filename": f"{TEST_DIR}/test.log",
+            "backup_dir": f"{TEST_DIR}/test_raw_data_backups",
+            "custom_run": False
+        },
+        "indicator": {
+            "wip_signal": True,
+            "export_start_date": "2020-08-01",
+            "static_file_dir": "./static",
+            "socrata_token": "test_token"
+        },
+        "validation": {
+            "common": {
+                "span_length": 14,
+                "min_expected_lag": {"all": "3"},
+                "max_expected_lag": {"all": "4"},
+            }
+        }
+    }
+    return copy.deepcopy(params)
+
+@pytest.fixture
+def params_w_patch(params):
+    params_copy = copy.deepcopy(params)
+    params_copy["common"]["custom_run"] = True
+    params_copy["patch"] = {
+            "patch_dir": f"{TEST_DIR}/patch_dir",
+            "source_dir": "test_source_dir",
+            "source_host": "host",
+            "user": "test_user",
+            "start_issue": "2023-01-01",
+            "end_issue": "2023-01-03",
+        }
+
+    return params_copy
+
+@pytest.fixture(scope="function")
+def run_as_module(params):
+    with patch('sodapy.Socrata.get') as mock_get:
+        def side_effect(*args, **kwargs):
+            if kwargs['offset'] == 0:
+                if DATASET_ID in args[0]:
+                    return TEST_DATA
+            else:
+                return []
+        mock_get.side_effect = side_effect
+        run_module(params)
+
+
+
+@pytest.fixture(scope="function")
+def run_as_module_hrr(params):
+    with patch('sodapy.Socrata.get') as mock_get, \
+         patch('delphi_nssp.run.GEOS', ["hrr"]):
+        def side_effect(*args, **kwargs):
+            if kwargs['offset'] == 0:
+                if DATASET_ID in args[0]:
+                    return HRR_TEST_DATA
+            else:
+                return []
+        mock_get.side_effect = side_effect
+        run_module(params)
+

nssp/tests/test_data/page.json

@@ -0,0 +1,200 @@
+[
+    {
+        "week_end": "2022-10-01T00:00:00.000",
+        "geography": "United States",
+        "county": "All",
+        "percent_visits_combined": "2.84",
+        "percent_visits_covid": "1.84",
+        "percent_visits_influenza": "0.48",
+        "percent_visits_rsv": "0.55",
+        "percent_visits_smoothed": "2.83",
+        "percent_visits_smoothed_covid": "2.07",
+        "percent_visits_smoothed_1": "0.34",
+        "percent_visits_smoothed_rsv": "0.44",
+        "ed_trends_covid": "Decreasing",
+        "ed_trends_influenza": "Increasing",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "All",
+        "hsa_counties": "All",
+        "hsa_nci_id": "All",
+        "fips": "0",
+        "trend_source": "United States",
+        "buildnumber": "2025-02-28"
+    },
+      {
+        "week_end": "2022-10-01T00:00:00.000",
+        "geography": "Colorado",
+        "county": "All",
+        "percent_visits_combined": "1.86",
+        "percent_visits_covid": "1.61",
+        "percent_visits_influenza": "0.1",
+        "percent_visits_rsv": "0.16",
+        "percent_visits_smoothed": "1.83",
+        "percent_visits_smoothed_covid": "1.61",
+        "percent_visits_smoothed_1": "0.1",
+        "percent_visits_smoothed_rsv": "0.12",
+        "ed_trends_covid": "Decreasing",
+        "ed_trends_influenza": "Increasing",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "All",
+        "hsa_counties": "All",
+        "hsa_nci_id": "All",
+        "fips": "8000",
+        "trend_source": "State",
+        "buildnumber": "2025-02-28"
+    },
+    {
+        "week_end": "2022-10-01T00:00:00.000",
+        "geography": "Colorado",
+        "county": "Broomfield",
+        "percent_visits_combined": "2.29",
+        "percent_visits_covid": "1.78",
+        "percent_visits_influenza": "0.11",
+        "percent_visits_rsv": "0.4",
+        "percent_visits_smoothed": "2.06",
+        "percent_visits_smoothed_covid": "1.7",
+        "percent_visits_smoothed_1": "0.07",
+        "percent_visits_smoothed_rsv": "0.32",
+        "ed_trends_covid": "No Change",
+        "ed_trends_influenza": "Increasing",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "Boulder, CO - Broomfield, CO",
+        "hsa_counties": "Boulder, Broomfield",
+        "hsa_nci_id": "795",
+        "fips": "8014",
+        "trend_source": "HSA",
+        "buildnumber": "2025-02-28"
+    },
+    {
+        "week_end": "2022-10-08T00:00:00.000",
+        "geography": "United States",
+        "county": "All",
+        "percent_visits_combined": "2.93",
+        "percent_visits_covid": "1.68",
+        "percent_visits_influenza": "0.68",
+        "percent_visits_rsv": "0.6",
+        "percent_visits_smoothed": "2.85",
+        "percent_visits_smoothed_covid": "1.85",
+        "percent_visits_smoothed_1": "0.49",
+        "percent_visits_smoothed_rsv": "0.53",
+        "ed_trends_covid": "Decreasing",
+        "ed_trends_influenza": "Increasing",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "All",
+        "hsa_counties": "All",
+        "hsa_nci_id": "All",
+        "fips": "0",
+        "trend_source": "United States",
+        "buildnumber": "2025-02-28"
+    },
+      {
+        "week_end": "2022-10-08T00:00:00.000",
+        "geography": "Colorado",
+        "county": "All",
+        "percent_visits_combined": "1.82",
+        "percent_visits_covid": "1.54",
+        "percent_visits_influenza": "0.09",
+        "percent_visits_rsv": "0.2",
+        "percent_visits_smoothed": "1.83",
+        "percent_visits_smoothed_covid": "1.58",
+        "percent_visits_smoothed_1": "0.1",
+        "percent_visits_smoothed_rsv": "0.16",
+        "ed_trends_covid": "Decreasing",
+        "ed_trends_influenza": "No Change",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "All",
+        "hsa_counties": "All",
+        "hsa_nci_id": "All",
+        "fips": "8000",
+        "trend_source": "State",
+        "buildnumber": "2025-02-28"
+    },
+        {
+        "week_end": "2022-10-08T00:00:00.000",
+        "geography": "Colorado",
+        "county": "Arapahoe",
+        "percent_visits_combined": "1.78",
+        "percent_visits_covid": "1.43",
+        "percent_visits_influenza": "0.12",
+        "percent_visits_rsv": "0.23",
+        "percent_visits_smoothed": "1.74",
+        "percent_visits_smoothed_covid": "1.4",
+        "percent_visits_smoothed_1": "0.12",
+        "percent_visits_smoothed_rsv": "0.25",
+        "ed_trends_covid": "Increasing",
+        "ed_trends_influenza": "Increasing",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "Denver (Denver), CO - Jefferson, CO",
+        "hsa_counties": "Adams, Arapahoe, Clear Creek, Denver, Douglas, Elbert, Gilpin, Grand, Jefferson, Park, Summit",
+        "hsa_nci_id": "688",
+        "fips": "8005",
+        "trend_source": "HSA",
+        "buildnumber": "2025-02-28"
+    },
+    {
+        "week_end": "2022-10-15T00:00:00.000",
+        "geography": "United States",
+        "county": "All",
+        "percent_visits_combined": "3.25",
+        "percent_visits_covid": "1.64",
+        "percent_visits_influenza": "0.9",
+        "percent_visits_rsv": "0.74",
+        "percent_visits_smoothed": "3.01",
+        "percent_visits_smoothed_covid": "1.72",
+        "percent_visits_smoothed_1": "0.69",
+        "percent_visits_smoothed_rsv": "0.63",
+        "ed_trends_covid": "Decreasing",
+        "ed_trends_influenza": "Increasing",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "All",
+        "hsa_counties": "All",
+        "hsa_nci_id": "All",
+        "fips": "0",
+        "trend_source": "United States",
+        "buildnumber": "2025-02-28"
+    },
+        {
+        "week_end": "2022-10-15T00:00:00.000",
+        "geography": "Colorado",
+        "county": "All",
+        "percent_visits_combined": "2.22",
+        "percent_visits_covid": "1.7",
+        "percent_visits_influenza": "0.14",
+        "percent_visits_rsv": "0.39",
+        "percent_visits_smoothed": "1.97",
+        "percent_visits_smoothed_covid": "1.62",
+        "percent_visits_smoothed_1": "0.11",
+        "percent_visits_smoothed_rsv": "0.25",
+        "ed_trends_covid": "No Change",
+        "ed_trends_influenza": "No Change",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "All",
+        "hsa_counties": "All",
+        "hsa_nci_id": "All",
+        "fips": "8000",
+        "trend_source": "State",
+        "buildnumber": "2025-02-28"
+    },
+    {
+        "week_end": "2022-10-15T00:00:00.000",
+        "geography": "Colorado",
+        "county": "Pueblo",
+        "percent_visits_combined": "3.24",
+        "percent_visits_covid": "2.97",
+        "percent_visits_influenza": "0.14",
+        "percent_visits_rsv": "0.14",
+        "percent_visits_smoothed": "2.99",
+        "percent_visits_smoothed_covid": "2.82",
+        "percent_visits_smoothed_1": "0.07",
+        "percent_visits_smoothed_rsv": "0.1",
+        "ed_trends_covid": "Increasing",
+        "ed_trends_influenza": "Increasing",
+        "ed_trends_rsv": "Increasing",
+        "hsa": "Pueblo (Pueblo), CO - Las Animas, CO",
+        "hsa_counties": "Huerfano, Las Animas, Pueblo",
+        "hsa_nci_id": "704",
+        "fips": "8101",
+        "trend_source": "HSA",
+        "buildnumber": "2025-02-28"
+    }
+]