Merge branch 'query-nested-strings' of https://github.com/cpcloud/pandas into cpcloud-query-nested-strings

Author: jreback

doc/source/release.rst

@@ -101,6 +101,8 @@ Bug Fixes
 - Bug in TimeGrouper/resample when presented with a non-monotonic DatetimeIndex would return invalid results. (:issue:`4161`)
 - Bug in index name propogation in TimeGrouper/resample (:issue:`4161`)
 - TimeGrouper has a more compatible API to the rest of the groupers (e.g. ``groups`` was missing) (:issue:`3881`)
+- Bug in ``pd.eval`` when parsing strings with possible tokens like ``'&'``
+  (:issue:`6351`)
 
 pandas 0.13.1
 -------------

pandas/computation/expr.py

@@ -252,7 +252,19 @@ def _replace_booleans(source):
     """Replace ``&`` with ``and`` and ``|`` with ``or`` so that bitwise
     precedence is changed to boolean precedence.
     """
-    return source.replace('|', ' or ').replace('&', ' and ')
+    res = []
+    g = tokenize.generate_tokens(StringIO(source).readline)
+    for toknum, tokval, _, _, _ in g:
+        if toknum == tokenize.OP:
+            if tokval == '&':
+                res.append((tokenize.NAME, 'and'))
+            elif tokval == '|':
+                res.append((tokenize.NAME, 'or'))
+            else:
+                res.append((toknum, tokval))
+        else:
+            res.append((toknum, tokval))
+    return tokenize.untokenize(res)
 
 
 def _replace_locals(source, local_symbol='@'):

pandas/io/tests/test_pytables.py

@@ -2199,7 +2199,7 @@ def test_remove_startstop(self):
         # GH #4835 and #6177
 
         with ensure_clean_store(self.path) as store:
-            
+
             wp = tm.makePanel()
 
             # start
@@ -2246,7 +2246,7 @@ def test_remove_startstop(self):
             result = store.select('wp6')
             expected = wp.reindex(major_axis=wp.major_axis)
             assert_panel_equal(result, expected)
-            
+
             # with where
             date = wp.major_axis.take(np.arange(0,30,3))
             crit = Term('major_axis=date')
@@ -2256,7 +2256,7 @@ def test_remove_startstop(self):
             result = store.select('wp7')
             expected = wp.reindex(major_axis=wp.major_axis-wp.major_axis[np.arange(0,20,3)])
             assert_panel_equal(result, expected)
-            
+
 
     def test_remove_crit(self):
 
@@ -4174,6 +4174,14 @@ def test_append_with_diff_col_name_types_raises_value_error(self):
                 with tm.assertRaises(ValueError):
                     store.append(name, d)
 
+    def test_query_with_nested_special_character(self):
+        df = DataFrame({'a': ['a', 'a', 'c', 'b', 'test & test', 'c' , 'b', 'e'],
+                        'b': [1, 2, 3, 4, 5, 6, 7, 8]})
+        expected = df[df.a == 'test & test']
+        with ensure_clean_store(self.path) as store:
+            store.append('test', df, format='table', data_columns=True)
+            result = store.select('test', 'a = "test & test"')
+        tm.assert_frame_equal(expected, result)
 
 def _test_sort(obj):
     if isinstance(obj, DataFrame):

pandas/tests/test_frame.py

@@ -12875,6 +12875,19 @@ def test_query_with_nested_string(self):
         for parser, engine in product(PARSERS, ENGINES):
             yield self.check_query_with_nested_strings, parser, engine
 
+    def check_query_with_nested_special_character(self, parser, engine):
+        skip_if_no_pandas_parser(parser)
+        tm.skip_if_no_ne(engine)
+        df = DataFrame({'a': ['a', 'b', 'test & test'],
+                        'b': [1, 2, 3]})
+        res = df.query('a == "test & test"', parser=parser, engine=engine)
+        expec = df[df.a == 'test & test']
+        tm.assert_frame_equal(res, expec)
+
+    def test_query_with_nested_special_character(self):
+        for parser, engine in product(PARSERS, ENGINES):
+            yield self.check_query_with_nested_special_character, parser, engine
+
     def check_query_lex_compare_strings(self, parser, engine):
         tm.skip_if_no_ne(engine=engine)
         import operator as opr