prod hotfix (#166)

* always run the data download
* same for flu
* add plain AR forecaster for covid
* fixes: update weights, update Makefile pull/push, fix a few filters

---------

Co-authored-by: Dmitry Shemetov <[email protected]>

Author: dsweber2

.gitignore

@@ -15,4 +15,6 @@ aux_data
 .netlify
 reports/*.html
 reports/report.md
-cache
+cache/
+data/
+.vscode/

Makefile

@@ -24,34 +24,38 @@ prod: prod-covid prod-flu update_site netlify
 
 submit-covid:
 	cd ../covid19-forecast-hub; \
-	git pull delphi main; \
+	git pull origin main; \
 	git add model-output/CMU-TimeSeries/*; \
+	git add model-output/CMU-climate_baseline/*; \
 	git commit -am "CMU-Delphi submission $(current_date)"; \
 	git push delphi main; \
 	gh pr create --title "CMU-TimeSeries $(current_date)" --repo cdcgov/covid19-forecast-hub
 
 submit-flu:
 	cd ../FluSight-forecast-hub; \
-	git pull delphi main; \
+	git pull origin main; \
 	git add model-output/CMU-TimeSeries/*; \
+	git add model-output/CMU-climate_baseline/*; \
 	git commit -am "CMU-Delphi submission $(current_date)"; \
-	git push delphi; \
+	git push delphi main; \
 	gh pr create --title "CMU-TimeSeries $(current_date)" --repo cdcepi/FluSight-forecast-hub
 
 submit-covid-dry:
 	cd ../covid19-forecast-hub; \
-	git pull delphi main; \
+	git pull origin main; \
 	git add model-output/CMU-TimeSeries/*; \
+	git add model-output/CMU-climate_baseline/*; \
 	git commit -am "CMU-Delphi submission $(current_date)"; \
 	git push delphi main; \
 	gh pr create --title "CMU-TimeSeries $(current_date)" --repo cdcgov/covid19-forecast-hub --dry-run
 
 submit-flu-dry:
 	cd ../FluSight-forecast-hub; \
-	git pull delphi main; \
+	git pull origin main; \
 	git add model-output/CMU-TimeSeries/*; \
+	git add model-output/CMU-climate_baseline/*; \
 	git commit -am "CMU-Delphi submission $(current_date)"; \
-	git push delphi; \
+	git push delphi main; \
 	gh pr create --title "CMU-TimeSeries $(current_date)" --repo cdcepi/FluSight-forecast-hub --dry-run
 
 submit: submit-covid submit-flu

covid_geo_exclusions.csv

@@ -8,6 +8,14 @@ forecast_date,forecaster,geo_value,weight
 "2024-10-01", "climate_base", "all", 2
 "2024-10-01", "climate_geo_agged", "all", 0.5
 "2024-10-01", "climate_quantile_extrapolated", "all", 0
+# feb 5
+2025-02-05, all, mp, 0
+2025-02-05, windowed_seasonal, all, 3
+2025-02-05, windowed_seasonal_extra_sources, all, 3
+2025-02-05, linear, all, 0.5
+2025-02-05, linearlog, all, 0
+2025-02-05, climate_base, all, 0
+2025-02-05, climate_geo_agged, all, 0.0
 # dec 4th
 "2024-12-04", "all", "mp", 0
 "2024-12-04", "linear", "all", 3

flu_geo_exclusions.csv

@@ -7,6 +7,14 @@ forecast_date,forecaster,geo_value,weight
 2024-10-01, climate_base, all, 0.5
 2024-10-01, climate_geo_agged, all, 0.25
 2024-10-01, climate_quantile_extrapolated, all, 0
+# feb 5
+2025-02-05, all, mp, 0
+2025-02-05, windowed_seasonal, all, 3
+2025-02-05, windowed_seasonal_extra_sources, all, 3
+2025-02-05, linear, all, 0.5
+2025-02-05, linearlog, all, 0
+2025-02-05, climate_base, all, 0
+2025-02-05, climate_geo_agged, all, 0.0
 # jan 8
 2025-01-08, all, mp, 0
 2025-01-08, windowed_seasonal, all, 3

scripts/covid_hosp_prod.R

@@ -39,6 +39,22 @@ forecaster_fns <- list2(
       geo_agg = TRUE
     )
   },
+  windowed_seasonal = function(epi_data, ahead, extra_data, ...) {
+    fcst <-
+      epi_data %>%
+      scaled_pop_seasonal(
+        outcome = "value",
+        ahead = ahead * 7,
+        ...,
+        seasonal_method = "none",
+        trainer = epipredict::quantile_reg(),
+        drop_non_seasons = TRUE,
+        pop_scaling = FALSE,
+        lags = list(c(0, 7))
+      ) %>%
+      mutate(target_end_date = target_end_date + 3)
+    fcst
+  },
   windowed_seasonal_extra_sources = function(epi_data, ahead, extra_data, ...) {
     fcst <-
       epi_data %>%
@@ -54,7 +70,9 @@ forecaster_fns <- list2(
         pop_scaling = FALSE,
         lags = list(c(0, 7), c(0, 7))
       ) %>%
-      mutate(target_end_date = target_end_date + 3)
+      mutate(target_end_date = target_end_date + 3) %>%
+      # Wyoming has no data for NSSP since July 2024
+      filter(geo_value != "wy")
     fcst
   }
 )
@@ -64,7 +82,7 @@ rlang::list2(
   tar_target(aheads, command = -1:3),
   tar_target(forecasters, command = indices),
   tar_target(
-    download_latest,
+    name = nhsn_latest_data,
     command = {
       if (wday(Sys.Date()) < 6 & wday(Sys.Date()) > 3) {
         # download from the preliminary data source from Wednesday to Friday
@@ -78,28 +96,9 @@ rlang::list2(
         filter(disease == "nhsn_covid") %>%
         select(-disease) %>%
         filter(geo_value %nin% insufficient_data_geos)
-      # if there's not already a result we need to save it no matter what
-      if (file.exists(here::here(".nhsn_covid_cache.parquet"))) {
-        previous_result <- qs::qread(here::here(".nhsn_covid_cache.parquet"))
-      } else
-      # if something is different, update the file
-      if (!isTRUE(all.equal(previous_result, most_recent_result))) {
-        qs::qsave(most_recent_result, here::here(".nhsn_covid_cache.parquet"))
-      } else {
-        qs::qsave(most_recent_result, here::here(".nhsn_covid_cache.parquet"))
-      }
-      NULL
-    },
-    description = "Download the result, and update the file only if it's actually different",
-    priority = 1,
-    cue = tar_cue(mode = "always")
-  ),
-  tar_change(
-    name = nhsn_latest_data,
-    command = {
-      qs::qread(here::here(".nhsn_covid_cache.parquet"))
+      most_recent_result
     },
-    change = tools::md5sum(here::here(".nhsn_covid_cache.parquet"))
+    cue = tar_cue("always")
   ),
   tar_target(
     name = nhsn_archive_data,
@@ -138,7 +137,7 @@ rlang::list2(
       }
     ),
     tar_target(
-      forecast_res,
+      name = forecast_res,
       command = {
         if (as.Date(forecast_generation_date_int) < Sys.Date()) {
           train_data <- nhsn_archive_data %>%
@@ -152,13 +151,13 @@ rlang::list2(
           train_data <-
             nhsn_latest_data %>%
             data_substitutions(disease = "covid") %>%
-            as_epi_df(as_of = as.Date(forecast_date_int))
+            as_epi_df(as_of = as.Date(forecast_date_int)) %>%
+            mutate(time_value = time_value - 3)
         }
         nssp <- current_nssp_archive %>%
           epix_as_of(min(forecast_date, current_nssp_archive$versions_end)) %>%
           mutate(time_value = time_value)
         attributes(train_data)$metadata$as_of <- as.Date(forecast_date_int)
-        print(names(forecaster_fns[forecasters]))
         train_data %>%
           forecaster_fns[[forecasters]](ahead = aheads, extra_data = nssp) %>%
           mutate(
@@ -196,7 +195,7 @@ rlang::list2(
           filter(geo_value %nin% geo_exclusions) %>%
           ungroup() %>%
           bind_rows(forecast_res %>%
-            filter(forecaster == "windowed_seasonal_extra_sources") %>%
+            filter(forecaster %in% c("windowed_seasonal", "windowed_seasonal_extra_sources")) %>%
             filter(forecast_date < target_end_date)) %>% # don't use for neg aheads
           group_by(geo_value, forecast_date, target_end_date, quantile) %>%
           summarize(value = mean(value, na.rm = TRUE), .groups = "drop") %>%

scripts/flu_hosp_prod.R

@@ -125,7 +125,7 @@ rlang::list2(
     }
   ),
   tar_target(
-    download_latest_nhsn,
+    nhsn_latest_data,
     command = {
       if (wday(Sys.Date()) < 6 & wday(Sys.Date()) > 3) {
         # download from the preliminary data source from Wednesday to Friday
@@ -148,30 +148,11 @@ rlang::list2(
         select(-version) %>%
         data_substitutions(disease = "flu") %>%
         as_epi_df(other_keys = "source", as_of = Sys.Date())
-      # if there's not already a result we need to save it no matter what
-      if (file.exists(here::here(".nhsn_flu_cache.parquet"))) {
-        previous_result <- qs::qread(here::here(".nhsn_flu_cache.parquet"))
-        # if something is different, update the file
-        # !isTRUE(all.equal) is true iff there's at least one difference
-        # can't use isFALSE(all.equal) because a bunch of strings are not, in fact, false
-        if (!isTRUE(all.equal(previous_result, most_recent_result))) {
-          qs::qsave(most_recent_result, here::here(".nhsn_flu_cache.parquet"))
-        }
-      } else {
-        qs::qsave(most_recent_result, here::here(".nhsn_flu_cache.parquet"))
-      }
-      NULL
+      most_recent_result
     },
     description = "Download the result, and update the file only if it's actually different",
     priority = 1,
-    cue = tar_cue(mode="always")
-  ),
-  tar_change(
-    name = nhsn_latest_data,
-    command = {
-      qs::qread(here::here(".nhsn_flu_cache.parquet"))
-    },
-    change = tools::md5sum(here::here(".nhsn_flu_cache.parquet"))
+    cue = tar_cue(mode = "always")
   ),
   tar_map(
     # Because targets relies on R metaprogramming, it loses the Date class.
@@ -275,38 +256,47 @@ rlang::list2(
           ensemble_linear_climate(aheads, other_weights = geo_forecasters_weights) %>%
           filter(geo_value %nin% geo_exclusions) %>%
           ungroup() %>%
-          # Ensemble with windowed_seasonal
-          bind_rows(forecast_res %>% filter(forecaster == "windowed_seasonal")) %>%
+          sort_by_quantile()
+      }
+    ),
+    tar_target(
+      name = ens_climate_linear_window_season,
+      command = {
+        climate_linear %>%
+          # Ensemble with windowed_seasonal and windowed_seasonal_extra_sources
+          bind_rows(forecast_res %>% filter(forecaster %in% c("windowed_seasonal", "windowed_seasonal_extra_sources"))) %>%
           group_by(geo_value, forecast_date, target_end_date, quantile) %>%
           summarize(value = mean(value, na.rm = TRUE), .groups = "drop") %>%
           sort_by_quantile()
       }
     ),
     tar_target(
-      name = ens_climate_linear_window_season,
+      name = ens_ar_only,
       command = {
         forecast_res %>%
-          # Apply the ahead-by-quantile weighting scheme
-          ensemble_linear_climate(aheads, other_weights = geo_forecasters_weights) %>%
-          filter(geo_value %nin% geo_exclusions) %>%
-          ungroup() %>%
-          # Ensemble with windowed_seasonal
-          bind_rows(forecast_res %>% filter(forecaster == "windowed_seasonal", forecaster == "windowed_seasonal_extra_sources")) %>%
+          filter(forecaster %in% c("windowed_seasonal", "windowed_seasonal_extra_sources")) %>%
           group_by(geo_value, forecast_date, target_end_date, quantile) %>%
           summarize(value = mean(value, na.rm = TRUE), .groups = "drop") %>%
           sort_by_quantile()
       }
     ),
     tar_target(
-      name = ens_climate_linear_window_season_ave_data,
+      name = climate_linear_modified,
       command = {
         forecast_res_modified %>%
           # Apply the ahead-by-quantile weighting scheme
           ensemble_linear_climate(aheads, other_weights = geo_forecasters_weights) %>%
           filter(geo_value %nin% geo_exclusions) %>%
           ungroup() %>%
+          sort_by_quantile()
+      }
+    ),
+    tar_target(
+      name = ens_climate_linear_window_season_modified,
+      command = {
+        climate_linear_modified %>%
           # Ensemble with windowed_seasonal
-          bind_rows(forecast_res_modified %>% filter(forecaster == "windowed_seasonal")) %>%
+          bind_rows(forecast_res_modified %>% filter(forecaster %in% c("windowed_seasonal", "windowed_seasonal_extra_sources"))) %>%
           group_by(geo_value, forecast_date, target_end_date, quantile) %>%
           summarize(value = mean(value, na.rm = TRUE), .groups = "drop") %>%
           sort_by_quantile()
@@ -316,7 +306,7 @@ rlang::list2(
       name = combo_ens_climate_linear_window_season,
       command = {
         inner_join(
-          ens_climate_linear_window_season, ens_climate_linear_window_season_ave_data,
+          ens_climate_linear_window_season, ens_climate_linear_window_season_modified,
           by = join_by(geo_value, forecast_date, target_end_date, quantile)
         ) %>%
           rowwise() %>%
@@ -332,10 +322,10 @@ rlang::list2(
       command = {
         bind_rows(
           forecast_res,
-          climate_linear %>% mutate(forecaster = "ensemble"),
+          climate_linear %>% mutate(forecaster = "climate_linear"),
+          ens_ar_only %>% mutate(forecaster = "ens_ar_only"),
           ens_climate_linear_window_season %>% mutate(forecaster = "ensemble_linclim_windowed_seasonal"),
-          ens_climate_linear_window_season_ave_data %>% mutate(forecaster = "ensemble_ave_data"),
-          combo_ens_climate_linear_window_season %>% mutate(forecaster = "ensemble_overall")
+          combo_ens_climate_linear_window_season %>% mutate(forecaster = "ensemble_combo")
         )
       }
     ),