Merge branch 'ndefries/backfill/speed2' into ndefries/backfill/speed-join-v-merge-order-matters

Author: nmdefries

backfill_corrections/delphiBackfillCorrection/R/model.R

@@ -208,22 +208,17 @@ get_model <- function(model_path, train_data, covariates, tau,
         " does not exist; training new model")
     }
     # Quantile regression
-    ## TODO: how does the speed compare using GLPK? Apparently it's faster on smaller
-    # models.
     obj <- quantile_lasso(as.matrix(train_data[covariates]),
                          train_data$log_value_target, tau = tau,
                          lambda = lambda, standardize = FALSE, lp_solver = lp_solver)
 
     # Save model to cache.
     create_dir_not_exist(dirname(model_path))
-    ## TODO: save() is fairly slow. Since we're not sharing the model files, can we
-    # use saveRDS() instead?
     save(obj, file=model_path)
   } else {
     # Load model from cache invisibly. Object has the same name as the original
     # model object, `obj`.
     msg_ts("Loading from ", model_path)
-    ## TODO: readRDS()
     load(model_path)
   }
 

backfill_corrections/delphiBackfillCorrection/R/preprocessing.R

@@ -93,7 +93,7 @@ get_7dav <- function(pivot_df, refd_col) {
     # Keep time values at the front
     pivot_df[, refd_col],
     # Compute moving average of all non-refd columns
-    RcppRoll::roll_mean(
+    roll_mean(
       as.matrix(pivot_df[, names(pivot_df)[names(pivot_df) != refd_col]]),
       7L, align = "right", fill = NA
     )
@@ -158,7 +158,7 @@ get_weekofmonth <- function(date) {
   month <- month(date)
   day <- day(date)
   firstdayofmonth <- as.numeric(format(make_date(year, month, 1), format="%u"))
-  n_days <- lubridate::days_in_month(date)
+  n_days <- days_in_month(date)
   n_weeks <- (n_days + firstdayofmonth - 1) %/% 7 + 1
   extra_check <- as.integer(n_weeks > 5)
   return (max((day + firstdayofmonth - 1) %/% 7 - extra_check, 0) %% 4 + 1)

backfill_corrections/delphiBackfillCorrection/R/utils.R

@@ -205,7 +205,7 @@ training_days_check <- function(issue_date, training_days) {
 get_populous_counties <- function() {
   return(
     covidcast::county_census %>%
-      dplyr::select(pop = POPESTIMATE2019, fips = FIPS) %>%
+      select(pop = POPESTIMATE2019, fips = FIPS) %>%
       # Drop megacounties (states)
       filter(!endsWith(fips, "000")) %>%
       arrange(desc(pop)) %>%

backfill_corrections/delphiBackfillCorrection/unit-tests/testthat/test-preprocessing.R

@@ -50,15 +50,15 @@ test_that("testing NA filling for missing udpates", {
 test_that("testing the calculation of 7-day moving average", {
   df_new <- fill_rows(fake_df, refd_col, lag_col, min_refd, max_refd, ref_lag)
   df <- fill_missing_updates(df_new, value_col, refd_col, lag_col)
-  df$issue_date <- df[[refd_col]] + df[[lag_col]]
+  df$issue_date <- as.Date(df[[refd_col]]) + df[[lag_col]]
   pivot_df <- df[order(df$issue_date, decreasing=FALSE), ] %>%
     pivot_wider(id_cols=refd_col, names_from="issue_date", 
                 values_from="value_raw")
   pivot_df[is.na(pivot_df)] = 0
   backfill_df <- get_7dav(pivot_df, refd_col)
 
 
-  output <- backfill_df[backfill_df[[refd_col]] == as.Date("2022-01-07"), "value_raw"]
+  output <- backfill_df[backfill_df[[refd_col]] == "2022-01-07", "value_raw"]
   expected <- colSums(pivot_df[, -1]) / 7
   expect_true(all(output == expected))
 })