Nancodes archiver: remove deleted file nan replacements

Author: dshemetov

_delphi_utils_python/delphi_utils/archive.py

@@ -254,26 +254,7 @@ def diff_exports(self) -> Tuple[Files, FileDiffMap, Files]:
                 new_issues_df.to_csv(diff_file, na_rep="NA")
                 common_diffs[after_file] = diff_file
 
-        export_csv_dtypes = {
-            "geo_id": str, "val": float, "se": float, "sample_size": float,
-            "missing_val": int, "missing_se": int, "missing_sample_size": int
-        }
-
-        # Replace deleted files with empty versions, but only if the cached version is not
-        # already empty
-        deleted_files_nanfilled = []
-        for deleted_file in deleted_files:
-            deleted_df = pd.read_csv(deleted_file, dtype=export_csv_dtypes)
-            print(
-                f"Diff deleted {deleted_file}; generating corresponding CSV with deleted rows."
-            )
-            deleted_df[["val", "se", "sample_size"]] = np.nan
-            deleted_df[["missing_val", "missing_se", "missing_sample_size"]] = Nans.DELETED
-            filename = join(self.export_dir, basename(deleted_file))
-            deleted_df.to_csv(filename, index=False)
-            deleted_files_nanfilled.append(filename)
-
-        return deleted_files_nanfilled, common_diffs, new_files
+        return deleted_files, common_diffs, new_files
 
     def archive_exports(self, exported_files: Files) -> Tuple[Files, Files]:
         """
@@ -331,13 +312,12 @@ def run(self):
         self.update_cache()
 
         # Diff exports, and make incremental versions
-        deleted_files, common_diffs, new_files = self.diff_exports()
+        _, common_diffs, new_files = self.diff_exports()
 
         # Archive changed, new, and emptied deleted files
         to_archive = [f for f, diff in common_diffs.items()
                       if diff is not None]
         to_archive += new_files
-        to_archive += deleted_files
         _, fails = self.archive_exports(to_archive)
 
         # Filter existing exports to exclude those that failed to archive

_delphi_utils_python/tests/test_archive.py

@@ -135,16 +135,6 @@ def test_diff_and_filter_exports(self, tmp_path):
             "missing_sample_size": [Nans.DELETED] + [Nans.NOT_MISSING] * 2,
         })
 
-        csv2_deleted = pd.DataFrame({
-            "geo_id": ["1"],
-            "val": [np.nan],
-            "se": [np.nan],
-            "sample_size": [np.nan],
-            "missing_val": [Nans.DELETED],
-            "missing_se": [Nans.DELETED],
-            "missing_sample_size": [Nans.DELETED],
-        })
-
         arch_diff = ArchiveDiffer(cache_dir, export_dir)
 
         # Test diff_exports
@@ -164,7 +154,7 @@ def test_diff_and_filter_exports(self, tmp_path):
         deleted_files, common_diffs, new_files = arch_diff.diff_exports()
 
         # Check return values
-        assert set(deleted_files) == {join(export_dir, "csv2.csv")}
+        assert set(deleted_files) == {join(cache_dir, "csv2.csv")}
         assert set(common_diffs.keys()) == {
             join(export_dir, f) for f in ["csv0.csv", "csv1.csv", "csv4.csv"]}
         assert set(new_files) == {join(export_dir, "csv3.csv")}
@@ -175,8 +165,7 @@ def test_diff_and_filter_exports(self, tmp_path):
         # Check filesystem for actual files
         assert set(listdir(export_dir)) == {
             "csv0.csv", "csv1.csv", "csv1.csv.diff",
-            "csv3.csv", "csv4.csv", "csv4.csv.diff",
-            "csv2.csv"
+            "csv3.csv", "csv4.csv", "csv4.csv.diff"
         }
         assert_frame_equal(
             pd.read_csv(join(export_dir, "csv1.csv.diff"), dtype=CSV_DTYPES),
@@ -194,11 +183,8 @@ def test_diff_and_filter_exports(self, tmp_path):
 
         arch_diff.filter_exports(common_diffs)
 
-        # Check exports directory just has incremental and deleted changes
-        assert set(listdir(export_dir)) == {"csv1.csv", "csv2.csv", "csv3.csv", "csv4.csv"}
-        assert_frame_equal(
-            pd.read_csv(join(export_dir, "csv2.csv"), dtype=CSV_DTYPES),
-            csv2_deleted)
+        # Check exports directory just has incremental changes
+        assert set(listdir(export_dir)) == {"csv1.csv", "csv3.csv", "csv4.csv"}
         assert_frame_equal(
             pd.read_csv(join(export_dir, "csv1.csv"), dtype=CSV_DTYPES),
             csv1_diff)
@@ -325,7 +311,7 @@ def test_run(self, tmp_path, s3_client):
             assert_frame_equal(pd.read_csv(body, dtype=CSV_DTYPES), df)
 
         # Check exports directory just has incremental changes
-        assert set(listdir(export_dir)) == {"csv1.csv", "csv2.csv", "csv3.csv", "csv4.csv"}
+        assert set(listdir(export_dir)) == {"csv1.csv", "csv3.csv", "csv4.csv"}
         csv1_diff = pd.DataFrame({
             "geo_id": ["3", "2", "4"],
             "val": [np.nan, 2.1, 4.0],
@@ -338,18 +324,6 @@ def test_run(self, tmp_path, s3_client):
         assert_frame_equal(
             pd.read_csv(join(export_dir, "csv1.csv"), dtype=CSV_DTYPES),
             csv1_diff)
-        csv2_deleted = pd.DataFrame({
-            "geo_id": ["1"],
-            "val": [np.nan],
-            "se": [np.nan],
-            "sample_size": [np.nan],
-            "missing_val": [Nans.DELETED],
-            "missing_se": [Nans.DELETED],
-            "missing_sample_size": [Nans.DELETED],
-        })
-        assert_frame_equal(
-            pd.read_csv(join(export_dir, "csv2.csv"), dtype=CSV_DTYPES),
-            csv2_deleted)
 
 
 class TestGitArchiveDiffer:
@@ -550,7 +524,7 @@ def test_run(self, tmp_path):
         original_branch.checkout()
 
         # Check exports directory just has incremental changes
-        assert set(listdir(export_dir)) == {"csv1.csv", "csv2.csv", "csv3.csv", "csv4.csv"}
+        assert set(listdir(export_dir)) == {"csv1.csv", "csv3.csv", "csv4.csv"}
         csv1_diff = pd.DataFrame({
             "geo_id": ["3", "2", "4"],
             "val": [np.nan, 2.1, 4.0],
@@ -563,19 +537,6 @@ def test_run(self, tmp_path):
         assert_frame_equal(
             pd.read_csv(join(export_dir, "csv1.csv"), dtype=CSV_DTYPES),
             csv1_diff)
-        csv2_deleted = pd.DataFrame({
-            "geo_id": ["1"],
-            "val": [np.nan],
-            "se": [np.nan],
-            "sample_size": [np.nan],
-            "missing_val": [Nans.DELETED],
-            "missing_se": [Nans.DELETED],
-            "missing_sample_size": [Nans.DELETED],
-        })
-        assert_frame_equal(
-            pd.read_csv(join(export_dir, "csv2.csv"), dtype=CSV_DTYPES),
-            csv2_deleted)
-
 
 
 class TestFromParams: