ENH: automagically created indicies (controlled by kw index=True/False passed to append/put)

Author: jreback

doc/source/io.rst

@@ -1144,7 +1144,7 @@ Queries are built up using a list of ``Terms`` (currently only **anding** of ter
 
 Indexing
 ~~~~~~~~
-You can create an index for a table with ``create_table_index`` after data is already in the table (after and ``append/put`` operation). Creating a table index is **highly** encouraged. This will speed your queries a great deal when you use a ``select`` with the indexed dimension as the ``where``. It is not automagically done now because you may want to index different axes than the default (except in the case of a DataFrame, where it almost always makes sense to index the ``index``.
+You can create/modify an index for a table with ``create_table_index`` after data is already in the table (after and ``append/put`` operation). Creating a table index is **highly** encouraged. This will speed your queries a great deal when you use a ``select`` with the indexed dimension as the ``where``. **Indexes are automagically created** on the indexables and any data columns you specify. This behavior can be turned off by passing ``index=False`` to ``append``.
 
 .. ipython:: python
 

pandas/io/pytables.py

@@ -507,7 +507,6 @@ def _write_to_group(self, key, value, table=False, append=False,
         wrapper(value)
         group._v_attrs.pandas_type = kind
         group._v_attrs.pandas_version = _version
-        #group._v_attrs.meta = getattr(value,'meta',None)
 
     def _write_series(self, group, series):
         self._write_index(group, 'index', series.index)
@@ -634,31 +633,37 @@ def _write_wide(self, group, panel):
     def _read_wide(self, group, where=None):
         return Panel(self._read_block_manager(group))
 
-    def _write_ndim_table(self, group, obj, append=False, comp=None, axes=None, **kwargs):
+    def _write_ndim_table(self, group, obj, append=False, comp=None, axes=None, index=True, **kwargs):
         if axes is None:
             axes = [1,2,3]
         t = create_table(self, group, typ = 'appendable_ndim')
         t.write(axes=axes, obj=obj,
                 append=append, compression=comp, **kwargs)
+        if index:
+            t.create_index()
 
     def _read_ndim_table(self, group, where=None, **kwargs):
         t = create_table(self, group, **kwargs)
         return t.read(where)
 
-    def _write_frame_table(self, group, df, append=False, comp=None, axes=None, **kwargs):
+    def _write_frame_table(self, group, df, append=False, comp=None, axes=None, index=True, **kwargs):
         if axes is None:
             axes = [0]
         t = create_table(self, group, typ = 'appendable_frame')
         t.write(axes=axes, obj=df, append=append, compression=comp, **kwargs)
+        if index:
+            t.create_index()
 
     _read_frame_table = _read_ndim_table
 
-    def _write_wide_table(self, group, panel, append=False, comp=None, axes=None, **kwargs):
+    def _write_wide_table(self, group, panel, append=False, comp=None, axes=None, index=True, **kwargs):
         if axes is None:
             axes = [1,2]
         t = create_table(self, group, typ = 'appendable_panel')
         t.write(axes=axes, obj=panel,
                 append=append, compression=comp, **kwargs)
+        if index:
+            t.create_index()
 
     _read_wide_table = _read_ndim_table
 
@@ -847,12 +852,7 @@ def _read_group(self, group, where=None, **kwargs):
         kind = group._v_attrs.pandas_type
         kind = _LEGACY_MAP.get(kind, kind)
         handler = self._get_handler(op='read', kind=kind)
-        v = handler(group, where, **kwargs)
-        #if v is not None:
-        #    meta = getattr(group._v_attrs,'meta',None)
-        #    if meta is not None:
-        #        v.meta = meta
-        return v
+        return handler(group, where, **kwargs)
 
     def _read_series(self, group, where=None):
         index = self._read_index(group, 'index')
@@ -1427,8 +1427,9 @@ def create_index(self, columns = None, optlevel = None, kind = None):
 
         if not self.infer_axes(): return
 
+        # index all indexables and data_columns
         if columns is None:
-            columns = [ self.index_axes[0].name ]
+            columns = [ a.cname for a in self.index_axes ] + [ v.cname for v in self.values_axes if v.name in set(self.data_columns) ]
         if not isinstance(columns, (tuple,list)):
             columns = [ columns ]
 

pandas/io/tests/test_pytables.py

@@ -430,6 +430,10 @@ def test_append_with_data_columns(self):
         self.store.append('df', df[2:])
         tm.assert_frame_equal(self.store['df'], df)
 
+        # check that we have indicies created
+        assert(self.store.handle.root.df.table.cols.index.is_indexed == True)
+        assert(self.store.handle.root.df.table.cols.B.is_indexed == True)
+
         # data column searching
         result = self.store.select('df', [ Term('B>0') ])
         expected = df[df.B>0]
@@ -441,14 +445,6 @@ def test_append_with_data_columns(self):
         expected = df_new[df_new.B>0]
         tm.assert_frame_equal(result, expected)
 
-        # index the columns
-        self.store.create_table_index('df', columns = ['B'])
-        result = self.store.select('df', [ Term('B>0'), Term('index','>',df.index[3]) ])
-        tm.assert_frame_equal(result, expected)
-
-        # check the index
-        assert(self.store.handle.root.df.table.cols.B.is_indexed == True)
-
         # data column selection with a string data_column
         df_new = df.copy()
         df_new['string'] = 'foo'
@@ -480,13 +476,21 @@ def test_append_with_data_columns(self):
         tm.assert_frame_equal(result, expected)
 
     def test_create_table_index(self):
+
+        # index=False
         wp = tm.makePanel()
-        self.store.append('p5', wp)
-        self.store.create_table_index('p5')
+        self.store.append('p5', wp, index=False)
+        self.store.create_table_index('p5', columns = ['major_axis'])
 
         assert(self.store.handle.root.p5.table.cols.major_axis.is_indexed == True)
         assert(self.store.handle.root.p5.table.cols.minor_axis.is_indexed == False)
 
+        # index=True
+        self.store.append('p5i', wp, index=True)
+
+        assert(self.store.handle.root.p5i.table.cols.major_axis.is_indexed == True)
+        assert(self.store.handle.root.p5i.table.cols.minor_axis.is_indexed == True)
+
         # default optlevels
         assert(self.store.handle.root.p5.table.cols.major_axis.index.optlevel == 6)
         assert(self.store.handle.root.p5.table.cols.major_axis.index.kind == 'medium')