BUG: correctly select on a multi-index even in the prescence of under specificed columsn spec (GH6169)

Author: jreback

doc/source/release.rst

@@ -175,6 +175,8 @@ Bug Fixes
   - Bug in ``HDFStore`` on appending a dataframe with multi-indexed columns to
     an existing table (:issue:`6167`)
   - Consistency with dtypes in setting an empty DataFrame (:issue:`6171`)
+  - Bug in  selecting on a multi-index ``HDFStore`` even in the prescence of under
+    specificed column spec (:issue:`6169`)
 
 pandas 0.13.0
 -------------

pandas/io/pytables.py

@@ -3289,6 +3289,12 @@ def create_axes(self, axes, obj, validate=True, nan_rep=None,
     def process_axes(self, obj, columns=None):
         """ process axes filters """
 
+        # make sure to include levels if we have them
+        if columns is not None and self.is_multi_index:
+            for n in self.levels:
+                if n not in columns:
+                    columns.insert(0, n)
+
         # reorder by any non_index_axes & limit to the select columns
         for axis, labels in self.non_index_axes:
             obj = _reindex_axis(obj, axis, labels, columns)
@@ -3305,6 +3311,12 @@ def process_filter(field, filt):
 
                         # see if the field is the name of an axis
                         if field == axis_name:
+
+                            # if we have a multi-index, then need to include
+                            # the levels
+                            if self.is_multi_index:
+                                filt = filt + Index(self.levels)
+
                             takers = op(axis_values, filt)
                             return obj.ix._getitem_axis(takers,
                                                         axis=axis_number)
@@ -3951,31 +3963,18 @@ def write(self, obj, data_columns=None, **kwargs):
         return super(AppendableMultiFrameTable, self).write(
             obj=obj, data_columns=data_columns, **kwargs)
 
-    def read(self, columns=None, **kwargs):
-        if columns is not None:
-            for n in self.levels:
-                if n not in columns:
-                    columns.insert(0, n)
-        df = super(AppendableMultiFrameTable, self).read(
-            columns=columns, **kwargs)
-        try:
-            df = df.set_index(self.levels)
-        except KeyError:
-            if kwargs.get('where') is not None and 'columns' in kwargs.get('where').expr:
-                raise KeyError(
-                "Indexes columns were not retrieved because you passed "
-                "a `where` argument  containing columns specification. "
-                "(see http://github.com/pydata/pandas/issues/6169), try passing "
-                "the columns specification through the `columns` keyword instead"
-                )
+    def read(self, **kwargs):
+
+        df = super(AppendableMultiFrameTable, self).read(**kwargs)
+        df = df.set_index(self.levels)
+
         # remove names for 'level_%d'
         df.index = df.index.set_names([
             None if self._re_levels.search(l) else l for l in df.index.names
         ])
 
         return df
 
-
 class AppendablePanelTable(AppendableTable):
 
     """ suppor the new appendable table formats """

pandas/io/tests/test_pytables.py

@@ -1674,7 +1674,7 @@ def make_index(names=None):
             tm.assert_frame_equal(store.select('df'),df)
 
     def test_select_columns_in_where(self):
-        
+
         # GH 6169
         # recreate multi-indexes when columns is passed
         # in the `where` argument
@@ -1687,19 +1687,22 @@ def test_select_columns_in_where(self):
         # With a DataFrame
         df = DataFrame(np.random.randn(10, 3), index=index,
                        columns=['A', 'B', 'C'])
-        
+
         with ensure_clean_store(self.path) as store:
             store.put('df', df, format='table')
-            tm.assert_frame_equal(store.select('df', where="columns=['A']"),df['A'],
-                                  check_index_type=True,check_column_type=True)
-        # With a Serie
+            expected = df[['A']]
+
+            tm.assert_frame_equal(store.select('df', columns=['A']), expected)
+
+            tm.assert_frame_equal(store.select('df', where="columns=['A']"), expected)
+
+        # With a Series
         s = Series(np.random.randn(10), index=index,
                    name='A')
         with ensure_clean_store(self.path) as store:
-            store.put('s', s)
-            tm.assert_frame_equal(store.select('s', where="columns=['A']"),s,
-                                  check_index_type=True,check_column_type=True)
-            
+            store.put('s', s, format='table')
+            tm.assert_series_equal(store.select('s', where="columns=['A']"),s)
+
     def test_pass_spec_to_storer(self):
 
         df = tm.makeDataFrame()