Merge pull request pandas-dev#532 from manahl/issue-531

Clean orphaned versions before prune (issue pandas-dev#531)

Author: aflag

CHANGES.md

@@ -1,6 +1,9 @@
 ## Changelog
 
-### 1.63
+### 1.64
+  * Bugfix: #531 arctic_prune_versions: clean broken snapshot references before pruning
+
+### 1.63 (2018-04-06)
   * Bugfix: #521 Clang 6.0 compiler support on macOS
   * Feature: #510 VersionStore: support multi column in pandas DataFrames
 

arctic/scripts/arctic_prune_versions.py

@@ -10,8 +10,12 @@
 logger = logging.getLogger(__name__)
 
 
-def prune_versions(lib, symbol, keep_mins):
-    lib._prune_previous_versions(symbol, keep_mins=keep_mins)
+def prune_versions(lib, symbols, keep_mins):
+    logger.info("Fixing snapshot pointers")
+    lib._cleanup_orphaned_versions(dry_run=False)
+    for symbol in symbols:
+        logger.info("Pruning %s" % symbol)
+        lib._prune_previous_versions(symbol, keep_mins=keep_mins)
 
 
 def main():
@@ -52,9 +56,7 @@ def main():
         symbols = lib.list_symbols(all_symbols=True)
         logger.info("Found %s symbols" % len(symbols))
 
-    for s in symbols:
-        logger.info("Pruning %s" % s)
-        prune_versions(lib, s, opts.keep_mins)
+    prune_versions(lib, symbols, opts.keep_mins)
     logger.info("Done")
 
 

tests/integration/scripts/test_prune_versions.py

@@ -1,3 +1,5 @@
+import time
+
 from mock import patch, ANY, call
 
 from arctic.auth import Credential
@@ -12,8 +14,7 @@ def test_prune_versions_symbol(mongo_host, library, library_name):
             patch('pymongo.database.Database.authenticate', return_value=True):
 
         run_as_main(mpv.main, '--host', mongo_host, '--library', library_name, '--symbols', 'sym1,sym2')
-        prune_versions.assert_has_calls([call(ANY, 'sym1', 10),
-                                         call(ANY, 'sym2', 10), ])
+        prune_versions.assert_has_calls([call(ANY, ['sym1', 'sym2'], 10)])
 
 
 def test_prune_versions_full(mongo_host, library, library_name):
@@ -37,3 +38,40 @@ def test_prune_versions_full(mongo_host, library, library_name):
         library.delete_snapshot('snap1')
         run_as_main(mpv.main, '--host', mongo_host, '--library', library_name, '--keep-mins', 0)
         assert [x['version'] for x in library.list_versions('symbol')] == [3]
+
+
+def test_keep_recent_snapshots(library):
+    library.write("cherry", "blob")
+    half_a_day_ago = time.time() - (3600 * 12.)
+    with patch('time.time', return_value=half_a_day_ago):
+        library.snapshot("snappy")
+    library._snapshots.delete_one({"name": "snappy"})
+
+    mpv.prune_versions(library, ["cherry"], 10)
+
+    assert len(library._versions.find_one({"symbol": "cherry"}).get("parent", [])) == 1
+
+
+def test_fix_broken_snapshot_references(library):
+    library.write("cherry", "blob")
+    one_day_ago = time.time() - (3600 * 24.) - 10  # make sure we are a few seconds before 24 hours
+    with patch('time.time', return_value=one_day_ago):
+        library.snapshot("snappy")
+    library._snapshots.delete_one({"name": "snappy"})
+
+    mpv.prune_versions(library, ["cherry"], 10)
+
+    assert library._versions.find_one({"symbol": "cherry"}).get("parent", []) == []
+
+
+def test_keep_only_one_version(library):
+    library.write("cherry", "blob")
+    library.write("cherry", "blob")
+    one_day_ago = time.time() - (3600 * 24.) - 10  # make sure we are a few seconds before 24 hours
+    with patch('time.time', return_value=one_day_ago):
+        library.snapshot("snappy")
+    library._snapshots.delete_one({"name": "snappy"})
+
+    mpv.prune_versions(library, ["cherry"], 0)
+
+    assert len(list(library._versions.find({"symbol": "cherry"}))) == 1