BUG: HDFStore didn't implement != correctly for string columns, GH 2973

Author: jreback

RELEASE.rst

@@ -106,6 +106,8 @@ pandas 0.11.0
       (e.g. store.df == store['df'])
     - Internally, change all variables to be private-like (now have leading
       underscore)
+    - fixes for query parsing to correctly interpret boolean and != (GH2849_, GH2973_)
+    - fixes for pathological case on SparseSeries with 0-len array and compression (GH2931_)
 
   - Bug showing up in applymap where some object type columns are converted (GH2909_)
     had an incorrect default in convert_objects
@@ -138,8 +140,11 @@ pandas 0.11.0
 .. _GH2845: https://github.com/pydata/pandas/issues/2845
 .. _GH2867: https://github.com/pydata/pandas/issues/2867
 .. _GH2807: https://github.com/pydata/pandas/issues/2807
+.. _GH2849: https://github.com/pydata/pandas/issues/2849
 .. _GH2898: https://github.com/pydata/pandas/issues/2898
 .. _GH2909: https://github.com/pydata/pandas/issues/2909
+.. _GH2931: https://github.com/pydata/pandas/issues/2931
+.. _GH2973: https://github.com/pydata/pandas/issues/2973
 
 
 pandas 0.10.1

pandas/core/panelnd.py

@@ -105,7 +105,7 @@ def _combine_with_constructor(self, other, func):
     klass._combine_with_constructor = _combine_with_constructor
 
     # set as NonImplemented operations which we don't support
-    for f in ['to_frame', 'to_excel', 'to_sparse', 'groupby', 'join', 'filter', 'dropna', 'shift', 'take']:
+    for f in ['to_frame', 'to_excel', 'to_sparse', 'groupby', 'join', 'filter', 'dropna', 'shift']:
         def func(self, *args, **kwargs):
             raise NotImplementedError
         setattr(klass, f, func)

pandas/io/pytables.py

@@ -2296,7 +2296,7 @@ def process_axes(self, obj, columns=None):
 
         # apply the selection filters (but keep in the same order)
         if self.selection.filter:
-            for field, filt in self.selection.filter:
+            for field, op, filt in self.selection.filter:
 
                 def process_filter(field, filt):
 
@@ -2306,9 +2306,8 @@ def process_filter(field, filt):
 
                         # see if the field is the name of an axis
                         if field == axis_name:
-                            ordd = axis_values & filt
-                            ordd = sorted(axis_values.get_indexer(ordd))
-                            return obj.reindex_axis(axis_values.take(ordd), axis=axis_number, copy=False)
+                            takers = op(axis_values,filt)
+                            return obj.ix._getitem_axis(takers,axis=axis_number)
 
                         # this might be the name of a file IN an axis
                         elif field in axis_values:
@@ -2320,7 +2319,8 @@ def process_filter(field, filt):
                             # hack until we support reversed dim flags
                             if isinstance(obj,DataFrame):
                                 axis_number = 1-axis_number
-                            return obj.ix._getitem_axis(values.isin(filt),axis=axis_number)
+                            takers = op(values,filt)
+                            return obj.ix._getitem_axis(takers,axis=axis_number)
 
                     raise Exception("cannot find the field [%s] for filtering!" % field)
 
@@ -2969,7 +2969,7 @@ def __init__(self, field, op=None, value=None, queryables=None):
         # backwards compatible
         if isinstance(field, dict):
             self.field = field.get('field')
-            self.op = field.get('op') or '='
+            self.op = field.get('op') or '=='
             self.value = field.get('value')
 
         # passed a term
@@ -2996,7 +2996,7 @@ def __init__(self, field, op=None, value=None, queryables=None):
                     self.op = op
                     self.value = value
                 else:
-                    self.op = '='
+                    self.op = '=='
                     self.value = op
 
         else:
@@ -3008,8 +3008,8 @@ def __init__(self, field, op=None, value=None, queryables=None):
             raise Exception("Could not create this term [%s]" % str(self))
 
         # = vs ==
-        if self.op == '==':
-            self.op = '='
+        if self.op == '=':
+            self.op = '=='
 
         # we have valid conditions
         if self.op in ['>', '>=', '<', '<=']:
@@ -3055,22 +3055,29 @@ def eval(self):
             values = [[v, v] for v in self.value]
 
         # equality conditions
-        if self.op in ['=', '!=']:
+        if self.op in ['==', '!=']:
+
+            # our filter op expression
+            if self.op == '!=':
+                filter_op = lambda axis, values: not axis.isin(values)
+            else:
+                filter_op = lambda axis, values: axis.isin(values)
+
 
             if self.is_in_table:
 
                 # too many values to create the expression?
                 if len(values) <= self._max_selectors:
                     self.condition = "(%s)" % ' | '.join(
-                        ["(%s == %s)" % (self.field, v[0]) for v in values])
+                        ["(%s %s %s)" % (self.field, self.op, v[0]) for v in values])
 
                 # use a filter after reading
                 else:
-                    self.filter = (self.field, Index([v[1] for v in values]))
+                    self.filter = (self.field, filter_op, Index([v[1] for v in values]))
 
             else:
 
-                self.filter = (self.field, Index([v[1] for v in values]))
+                self.filter = (self.field, filter_op, Index([v[1] for v in values]))
 
         else:
 

pandas/io/tests/test_pytables.py

@@ -1673,7 +1673,7 @@ def test_select_dtypes(self):
             expected = df[df.ts >= Timestamp('2012-02-01')]
             tm.assert_frame_equal(expected, result)
 
-            # bool columns
+            # bool columns (GH #2849)
             df = DataFrame(np.random.randn(5,2), columns =['A','B'])
             df['object'] = 'foo'
             df.ix[4:5,'object'] = 'bar'
@@ -1801,6 +1801,54 @@ def test_frame_select(self):
             # self.assertRaises(Exception, store.select,
             #                  'frame', [crit1, crit2])
 
+    def test_string_select(self):
+
+        # GH 2973
+
+        df = tm.makeTimeDataFrame()
+
+        with ensure_clean(self.path) as store:
+
+
+            # test string ==/!=
+
+            df['x'] = 'none'
+            df.ix[2:7,'x'] = ''
+
+            store.append('df',df,data_columns=['x'])
+
+            result = store.select('df',Term('x=none'))
+            expected = df[df.x == 'none']
+            assert_frame_equal(result,expected)
+
+            result = store.select('df',Term('x!=none'))
+            expected = df[df.x != 'none']
+            assert_frame_equal(result,expected)
+
+            df2 = df.copy()
+            df2.x[df2.x==''] = np.nan
+
+            from pandas import isnull
+            store.append('df2',df2,data_columns=['x'])
+            result = store.select('df2',Term('x!=none'))
+            expected = df2[isnull(df2.x)]
+            assert_frame_equal(result,expected)
+
+            # int ==/!=
+            df['int'] = 1
+            df.ix[2:7,'int'] = 2
+
+            store.append('df3',df,data_columns=['int'])
+
+            result = store.select('df3',Term('int=2'))
+            expected = df[df.int==2]
+            assert_frame_equal(result,expected)
+
+            result = store.select('df3',Term('int!=2'))
+            expected = df[df.int!=2]
+            assert_frame_equal(result,expected)
+
+
     def test_unique(self):
 
         df = tm.makeTimeDataFrame()