ERR/API: disallow local references in top-level calls to eval

Author: cpcloud

pandas/compat/__init__.py

@@ -54,7 +54,7 @@
     import pickle as cPickle
     import http.client as httplib
 
-from chainmap import DeepChainMap
+from pandas.compat.chainmap import DeepChainMap
 
 
 if PY3:

pandas/computation/eval.py

@@ -4,8 +4,9 @@
 """
 
 import sys
+import tokenize
 from pandas.core import common as com
-from pandas.computation.expr import Expr, _parsers
+from pandas.computation.expr import Expr, _parsers, tokenize_string
 from pandas.computation.scope import _ensure_scope
 from pandas.compat import DeepChainMap, builtins
 from pandas.computation.engines import _engines
@@ -118,6 +119,24 @@ def _convert_expression(expr):
     return s
 
 
+def _check_for_locals(expr, stack_level, parser):
+    at_top_of_stack = stack_level == 0
+    not_pandas_parser = parser != 'pandas'
+
+    if not_pandas_parser:
+        msg = "The '@' prefix is only supported by the pandas parser"
+    elif at_top_of_stack:
+        msg = ("The '@' prefix is not allowed in "
+               "top-level eval calls, please refer to "
+               "your variables by name without the '@' "
+               "prefix")
+
+    if at_top_of_stack or not_pandas_parser:
+        for toknum, tokval, _, _, _ in tokenize_string(expr):
+            if toknum == tokenize.OP and tokval == '@':
+                raise SyntaxError(msg)
+
+
 def eval(expr, parser='pandas', engine='numexpr', truediv=True,
          local_dict=None, global_dict=None, resolvers=(), level=0,
          target=None):
@@ -200,6 +219,7 @@ def eval(expr, parser='pandas', engine='numexpr', truediv=True,
     _check_engine(engine)
     _check_parser(parser)
     _check_resolvers(resolvers)
+    _check_for_locals(expr, level, parser)
 
     # get our (possibly passed-in) scope
     level += 1

pandas/computation/expr.py

@@ -23,13 +23,16 @@
 from pandas.computation.scope import Scope, _ensure_scope
 
 
+def tokenize_string(s):
+    return tokenize.generate_tokens(StringIO(s).readline)
+
+
 def _rewrite_assign(source):
     """Rewrite the assignment operator for PyTables expression that want to use
     ``=`` as a substitute for ``==``.
     """
     res = []
-    g = tokenize.generate_tokens(StringIO(source).readline)
-    for toknum, tokval, _, _, _ in g:
+    for toknum, tokval, _, _, _ in tokenize_string(source):
         res.append((toknum, '==' if tokval == '=' else tokval))
     return tokenize.untokenize(res)
 
@@ -39,8 +42,7 @@ def _replace_booleans(source):
     precedence is changed to boolean precedence.
     """
     res = []
-    g = tokenize.generate_tokens(StringIO(source).readline)
-    for toknum, tokval, _, _, _ in g:
+    for toknum, tokval, _, _, _ in tokenize_string(source):
         if toknum == tokenize.OP:
             if tokval == '&':
                 res.append((tokenize.NAME, 'and'))
@@ -54,7 +56,7 @@ def _replace_booleans(source):
 
 
 def _replace_locals(source, local_symbol='@'):
-    """Replace local variables with a syntacticall valid name."""
+    """Replace local variables with a syntactically valid name."""
     return source.replace(local_symbol, _LOCAL_TAG)
 
 

pandas/computation/pytables.py

@@ -11,7 +11,7 @@
 from pandas.core.base import StringMixin
 import pandas.core.common as com
 from pandas.computation import expr, ops
-from pandas.computation.ops import is_term
+from pandas.computation.ops import is_term, UndefinedVariableError
 from pandas.computation.scope import _ensure_scope
 from pandas.computation.expr import BaseExprVisitor
 from pandas.computation.common import _ensure_decoded
@@ -48,7 +48,10 @@ def _resolve_name(self):
             return self.name
 
         # resolve the rhs (and allow it to be None)
-        return self.env.resolve(self.name, is_local=False)
+        try:
+            return self.env.resolve(self.name, is_local=False)
+        except UndefinedVariableError:
+            return self.name
 
     @property
     def value(self):

pandas/computation/scope.py

@@ -10,7 +10,7 @@
 import pprint
 
 import pandas as pd
-from pandas.compat import DeepChainMap, map
+from pandas.compat import DeepChainMap, map, StringIO
 from pandas.core import common as com
 from pandas.core.base import StringMixin
 from pandas.computation.ops import UndefinedVariableError, _LOCAL_TAG
@@ -117,11 +117,11 @@ def __init__(self, level, global_dict=None, local_dict=None, resolvers=(),
             # shallow copy here because we don't want to replace what's in
             # scope when we align terms (alignment accesses the underlying
             # numpy array of pandas objects)
+            self.scope = self.scope.new_child((global_dict or
+                                               frame.f_globals).copy())
             if not isinstance(local_dict, Scope):
                 self.scope = self.scope.new_child((local_dict or
                                                    frame.f_locals).copy())
-            self.scope = self.scope.new_child((global_dict or
-                                               frame.f_globals).copy())
         finally:
             del frame
 
@@ -132,8 +132,8 @@ def __init__(self, level, global_dict=None, local_dict=None, resolvers=(),
         self.temps = {}
 
     def __unicode__(self):
-        scope_keys = _get_pretty_string(self.scope.keys())
-        res_keys = _get_pretty_string(self.resolvers.keys())
+        scope_keys = _get_pretty_string(list(self.scope.keys()))
+        res_keys = _get_pretty_string(list(self.resolvers.keys()))
         return '%s(scope=%s, resolvers=%s)' % (type(self).__name__, scope_keys,
                                                res_keys)
 

pandas/computation/tests/test_eval.py

@@ -1556,6 +1556,25 @@ def test_invalid_numexpr_version():
         yield check_invalid_numexpr_version, engine, parser
 
 
+def check_invalid_local_variable_reference(engine, parser):
+    tm.skip_if_no_ne(engine)
+
+    a, b = 1, 2
+    exprs = 'a + @b', '@a + b', '@a + @b'
+    for expr in exprs:
+        if parser != 'pandas':
+            with tm.assertRaisesRegexp(SyntaxError, "The '@' prefix is only"):
+                pd.eval(exprs, engine=engine, parser=parser)
+        else:
+            with tm.assertRaisesRegexp(SyntaxError, "The '@' prefix is not"):
+                pd.eval(exprs, engine=engine, parser=parser)
+
+
+def test_invalid_local_variable_reference():
+    for engine, parser in ENGINES_PARSERS:
+        yield check_invalid_local_variable_reference, engine, parser
+
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                    exit=False)

pandas/io/pytables.py

@@ -81,7 +81,7 @@ def _ensure_term(where, scope_level):
         where = [w if not maybe_expression(w) else Term(w, scope_level=level)
                  for w in where if w is not None]
     elif maybe_expression(where):
-        where = Term(where, level)
+        where = Term(where, scope_level=level)
     return where
 
 

pandas/tests/test_frame.py

@@ -12116,7 +12116,6 @@ def test_isin_dupe_self(self):
         expected.iloc[1, 1] = True
         assert_frame_equal(result, expected)
 
-
     def test_isin_against_series(self):
         df = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [2, np.nan, 4, 4]},
                           index=['a', 'b', 'c', 'd'])
@@ -12249,6 +12248,7 @@ def test_empty_frame_dtypes_ftypes(self):
                                                             ('b', 'bool:dense'),
                                                             ('c', 'float64:dense')])))
 
+
 def skip_if_no_ne(engine='numexpr'):
     if engine == 'numexpr':
         try:
@@ -12705,16 +12705,16 @@ def test_nested_scope(self):
         result = df.query('(@df > 0) & (@df2 > 0)', engine=engine, parser=parser)
         assert_frame_equal(result, expected)
 
-        result = pd.eval('@df[@df > 0 and @df2 > 0]', engine=engine,
+        result = pd.eval('df[df > 0 and df2 > 0]', engine=engine,
                          parser=parser)
         assert_frame_equal(result, expected)
 
-        result = pd.eval('@df[@df > 0 and @df2 > 0 and @df[@df > 0] > 0]',
+        result = pd.eval('df[df > 0 and df2 > 0 and df[df > 0] > 0]',
                          engine=engine, parser=parser)
         expected = df[(df > 0) & (df2 > 0) & (df[df > 0] > 0)]
         assert_frame_equal(result, expected)
 
-        result = pd.eval('@df[(@df>0) & (@df2>0)]', engine=engine, parser=parser)
+        result = pd.eval('df[(df>0) & (df2>0)]', engine=engine, parser=parser)
         expected = df.query('(@df>0) & (@df2>0)', engine=engine, parser=parser)
         assert_frame_equal(result, expected)
 
@@ -12874,6 +12874,7 @@ def test_query_builtin(self):
         result = df.query('sin > 5', engine=engine, parser=parser)
         tm.assert_frame_equal(expected, result)
 
+
 class TestDataFrameQueryPythonPython(TestDataFrameQueryNumExprPython):
 
     @classmethod
@@ -12894,6 +12895,7 @@ def test_query_builtin(self):
         result = df.query('sin > 5', engine=engine, parser=parser)
         tm.assert_frame_equal(expected, result)
 
+
 PARSERS = 'python', 'pandas'
 ENGINES = 'python', 'numexpr'