Add quota to chunkstore and fix writes with missing column names (pandas-dev#261)

Author: bmoscon

arctic/chunkstore/chunkstore.py

@@ -6,6 +6,7 @@
 from pandas import DataFrame, Series
 from six.moves import xrange
 from itertools import groupby
+from pymongo.errors import OperationFailure
 
 from ..decorators import mongo_retry
 from .._util import indent
@@ -248,6 +249,8 @@ def write(self, symbol, item, chunker=DateChunker(), **kwargs):
         if not isinstance(item, (DataFrame, Series)):
             raise Exception("Can only chunk DataFrames and Series")
 
+        self._arctic_lib.check_quota()
+
         previous_shas = []
         doc = {}
 
@@ -319,6 +322,8 @@ def __update(self, sym, item, combine_method=None, chunk_range=None):
         if not isinstance(item, (DataFrame, Series)):
             raise Exception("Can only chunk DataFrames and Series")
 
+        self._arctic_lib.check_quota()
+
         symbol = sym[SYMBOL]
 
         if chunk_range is not None:
@@ -535,3 +540,32 @@ def reverse_iterator(self, symbol, chunk_range=None):
 
         for chunk in chunks:
             yield self.read(symbol, chunk_range=c.to_range(chunk[0], chunk[1]))
+            
+    def stats(self):
+        """
+        Return storage statistics about the library
+
+        Returns
+        -------
+        dictionary of storage stats
+        """
+
+        res = {}
+        db = self._collection.database
+        conn = db.connection
+        res['sharding'] = {}
+        try:
+            sharding = conn.config.databases.find_one({'_id': db.name})
+            if sharding:
+                res['sharding'].update(sharding)
+            res['sharding']['collections'] = list(conn.config.collections.find({'_id': {'$regex': '^' + db.name + "\..*"}}))
+        except OperationFailure:
+            # Access denied
+            pass
+        res['dbstats'] = db.command('dbstats')
+        res['chunks'] = db.command('collstats', self._collection.name)
+        res['symbols'] = db.command('collstats', self._symbols.name)
+        res['totals'] = {'count': res['chunks']['count'],
+                         'size': res['chunks']['size'] + res['symbols']['size'],
+                         }
+        return res

arctic/serialization/numpy_arrays.py

@@ -150,6 +150,9 @@ def serialize(self, df):
         else:
             dtype = 'dataframe'
 
+        if (len(df.index.names) > 1 and None in df.index.names) or None in list(df.columns.values):
+            raise Exception("All columns and indexes must be named")
+
         if df.index.names != [None]:
             index = df.index.names
             df = df.reset_index()

setup.py

@@ -99,8 +99,8 @@ def run_tests(self):
     tests_require=["mock",
                    "mockextras",
                    "pytest<=2.9.2",
-                   "pytest-cov",
-                   "pytest-dbfixtures>=0.15.0",
+                   "pytest-cov<=2.3.1",
+                   "pytest-dbfixtures==0.15.0",
                    "pytest-timeout",
                    "pytest-xdist",
                    ],

tests/integration/chunkstore/test_chunkstore.py

@@ -1150,3 +1150,25 @@ def test_iterators(chunkstore_lib):
     dr = DateRange(dt(2016, 1, 2), dt(2016, 1, 2))
     assert(len(list(chunkstore_lib.iterator('test_df', chunk_range=dr))) == 1)
     assert(len(list(chunkstore_lib.reverse_iterator('test_df', chunk_range=dr))) == 1)
+
+
+def test_unnamed_colums(chunkstore_lib):
+    df = DataFrame(data={'data': [1, 2, 3]},
+                   index=MultiIndex.from_tuples([(dt(2016, 1, 1), 1),
+                                                 (dt(2016, 1, 2), 1),
+                                                 (dt(2016, 1, 3), 1)],
+                                                names=['date', None])
+                   )
+    with pytest.raises(Exception) as e:
+        chunkstore_lib.write('test_df', df, chunk_size='D')
+    assert('must be named' in str(e))
+
+    df = DataFrame(data={None: [1, 2, 3]},
+                   index=MultiIndex.from_tuples([(dt(2016, 1, 1), 1),
+                                                 (dt(2016, 1, 2), 1),
+                                                 (dt(2016, 1, 3), 1)],
+                                                names=['date', 'id'])
+                   )
+    with pytest.raises(Exception) as e:
+        chunkstore_lib.write('test_df', df, chunk_size='D')
+    assert('must be named' in str(e))