Merge pull request #1181 from cmu-delphi/remove-jhu-archivediffer

Remove ArchiveDiffer from JHU run()

Author: krivard

jhu/delphi_jhu/run.py

@@ -12,7 +12,6 @@
 import numpy as np
 from delphi_utils import (
     create_export_csv,
-    S3ArchiveDiffer,
     Smoother,
     GeoMapper,
     get_structured_logger,
@@ -89,18 +88,6 @@ def run_module(params: Dict[str, Any]):
         __name__, filename=params["common"].get("log_filename"),
         log_exceptions=params["common"].get("log_exceptions", True))
 
-    if "archive" in params:
-        arch_diff = S3ArchiveDiffer(
-            params["archive"]["cache_dir"],
-            export_dir,
-            params["archive"]["bucket_name"],
-            "jhu",
-            params["archive"]["aws_credentials"],
-        )
-        arch_diff.update_cache()
-    else:
-        arch_diff = None
-
     gmpr = GeoMapper()
     dfs = {metric: pull_jhu_data(base_url, metric, gmpr) for metric in METRICS}
     for metric, geo_res, sensor, smoother in product(
@@ -144,25 +131,6 @@ def run_module(params: Dict[str, Any]):
             oldest_final_export_date = min(
                 oldest_final_export_date, max(exported_csv_dates))
 
-    if arch_diff is not None:
-        # Diff exports, and make incremental versions
-        _, common_diffs, new_files = arch_diff.diff_exports()
-
-        # Archive changed and new files only
-        to_archive = [f for f, diff in common_diffs.items() if diff is not None]
-        to_archive += new_files
-        _, fails = arch_diff.archive_exports(to_archive)
-
-        # Filter existing exports to exclude those that failed to archive
-        succ_common_diffs = {
-            f: diff for f, diff in common_diffs.items() if f not in fails
-        }
-        arch_diff.filter_exports(succ_common_diffs)
-
-        # Report failures: someone should probably look at them
-        for exported_file in fails:
-            print(f"Failed to archive '{exported_file}'")
-
     elapsed_time_in_seconds = round(time.time() - start_time, 2)
     max_lag_in_days = None
     formatted_oldest_final_export_date = None