From 510a11e50fb51717894946dee875c970c3949f14 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Tue, 11 Apr 2023 12:30:54 -0400
Subject: [PATCH 1/5] log when geo-splitting is over

---
 backfill_corrections/delphiBackfillCorrection/R/main.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backfill_corrections/delphiBackfillCorrection/R/main.R b/backfill_corrections/delphiBackfillCorrection/R/main.R
index 802777f51..69d37fa67 100644
--- a/backfill_corrections/delphiBackfillCorrection/R/main.R
+++ b/backfill_corrections/delphiBackfillCorrection/R/main.R
@@ -60,6 +60,7 @@ run_backfill <- function(df, params,
     msg_ts("Splitting data into geo groups")
     group_dfs <- group_split(df, geo_value)
 
+    msg_ts("Beginning training and/or testing...")
     # Build model for each location
     apply_fn <- ifelse(params$parallel, mclapply, lapply)
     result <- apply_fn(group_dfs, function(subdf) {

From 02c8c40130173f9abd9e7f41eb77ff2f10059078 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Tue, 11 Apr 2023 12:32:01 -0400
Subject: [PATCH 2/5] remove everything in output dir

---
 backfill_corrections/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backfill_corrections/Makefile b/backfill_corrections/Makefile
index e58418dbc..8514e8453 100644
--- a/backfill_corrections/Makefile
+++ b/backfill_corrections/Makefile
@@ -124,7 +124,7 @@ standardize-dirs:
 	$(PYTHON) -m delphi_utils set export_dir $(EXPORT_DIR)
 
 clean:
-	rm -f $(USR_EXPORT_DIR)/*.csv.gz
+	rm -rf $(USR_EXPORT_DIR)/*
 
 coverage:
 	Rscript -e 'covr::package_coverage("delphiBackfillCorrection")'

From dd671bfdc771128de263f534590897316973d85e Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Tue, 11 Apr 2023 12:34:17 -0400
Subject: [PATCH 3/5] increase shared memory size to support parallel
 prediction generation

---
 backfill_corrections/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backfill_corrections/Makefile b/backfill_corrections/Makefile
index 8514e8453..7117b3849 100644
--- a/backfill_corrections/Makefile
+++ b/backfill_corrections/Makefile
@@ -81,6 +81,7 @@ run:
 		-v "`realpath $(USR_CACHE_DIR)`:/backfill_corrections/${CACHE_DIR}" \
 		-v "${PWD}"/params.json:/backfill_corrections/params.host.json \
 		--env GRB_LICENSE_FILE=$(GRB_LICENSE_FILE) \
+		--shm-size=5gb \
 		-it "${DOCKER_IMAGE}:${DOCKER_TAG}" \
 		/bin/bash -c "cp params.host.json params.json && make gurobi.lic && make standardize-dirs && make run-local OPTIONS=\"${OPTIONS}\" LOG_FILE=${LOG_FILE}"
 

From 8875b40271ebbc552877c313c0539a276404822d Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Tue, 11 Apr 2023 12:35:59 -0400
Subject: [PATCH 4/5] check logs for lack of shared memory error

---
 backfill_corrections/Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backfill_corrections/Makefile b/backfill_corrections/Makefile
index 7117b3849..62a2f702b 100644
--- a/backfill_corrections/Makefile
+++ b/backfill_corrections/Makefile
@@ -67,6 +67,8 @@ run-local: setup-dirs
 	grep "backfill correction completed successfully" $(LOG_FILE)
 	grep "scheduled core" $(LOG_FILE) ; \
 	[ "$$?" -eq 1 ]
+	grep "SIGBUS" $(LOG_FILE) ; \
+	[ "$$?" -eq 1 ]
 
 gurobi.lic:
 	@echo WLSACCESSID=$(GRB_WLSACCESSID) >> $(GRB_LICENSE_FILE)

From b55abe8face30cbaafa51d0ee8f124694ea1b8b5 Mon Sep 17 00:00:00 2001
From: Nat DeFries <42820733+nmdefries@users.noreply.github.com>
Date: Tue, 11 Apr 2023 17:34:16 -0400
Subject: [PATCH 5/5] lower shared memory allocation

---
 backfill_corrections/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backfill_corrections/Makefile b/backfill_corrections/Makefile
index 62a2f702b..68abff228 100644
--- a/backfill_corrections/Makefile
+++ b/backfill_corrections/Makefile
@@ -83,7 +83,7 @@ run:
 		-v "`realpath $(USR_CACHE_DIR)`:/backfill_corrections/${CACHE_DIR}" \
 		-v "${PWD}"/params.json:/backfill_corrections/params.host.json \
 		--env GRB_LICENSE_FILE=$(GRB_LICENSE_FILE) \
-		--shm-size=5gb \
+		--shm-size=2gb \
 		-it "${DOCKER_IMAGE}:${DOCKER_TAG}" \
 		/bin/bash -c "cp params.host.json params.json && make gurobi.lic && make standardize-dirs && make run-local OPTIONS=\"${OPTIONS}\" LOG_FILE=${LOG_FILE}"