ENH: Add implementation for backtick quoting (pandas-dev#6508).

This only works for parser=pandas and engine=numexpr
It works by replacing any backtick quoted variables to a clean version.
    For this, see: pandas/core/common.py::clean_column_name_with_spaces
This happens before the query is passed and by changing the names
of the localdict before it passed to numexpr.

Author: hwalinga

pandas/core/common.py

@@ -66,6 +66,14 @@ def consensus_name_attr(objs):
     return name
 
 
+def clean_column_name_with_spaces(name):
+    """Check if name contains any spaces, if it contains any spaces
+    the spaces will be removed and an underscore suffix is added."""
+    if not isinstance(name, str) or " " not in name:
+        return name
+    return name.replace(" ", "_") + "_"
+
+
 def maybe_box(indexer, values, obj, key):
 
     # if we have multiples coming back, box em

pandas/core/computation/expr.py

@@ -3,6 +3,7 @@
 
 import ast
 from functools import partial
+import re
 import tokenize
 
 import numpy as np
@@ -102,6 +103,26 @@ def _replace_locals(tok):
     return toknum, tokval
 
 
+def _replace_spaces_backtickvariables(source):
+    """Clean up a column name if surrounded by backticks.
+
+    Parameters
+    ----------
+    source : str
+        str corresponding to an expression
+
+    Return
+    ------
+    s : str
+        Anything that was surrounded by spaces in source is now cleanup up.
+        This is done so that any collumn name with spaces still can be used
+        in query and eval.
+    """
+    return re.sub(r'`(.*?)`',
+                  lambda m: com.clean_column_name_with_spaces(m.group(1)),
+                  source)
+
+
 def _compose2(f, g):
     """Compose 2 callables"""
     return lambda *args, **kwargs: f(g(*args, **kwargs))
@@ -114,7 +135,7 @@ def _compose(*funcs):
 
 
 def _preparse(source, f=_compose(_replace_locals, _replace_booleans,
-                                 _rewrite_assign)):
+                                 _rewrite_assign), g=lambda x: x):
     """Compose a collection of tokenization functions
 
     Parameters
@@ -127,6 +148,9 @@ def _preparse(source, f=_compose(_replace_locals, _replace_booleans,
         to the composition of ``_rewrite_assign``, ``_replace_booleans``, and
         ``_replace_locals``.
 
+    g : callable
+        This takes a source string and returns an altered one.
+
     Returns
     -------
     s : str
@@ -139,7 +163,8 @@ def _preparse(source, f=_compose(_replace_locals, _replace_booleans,
     the ``tokenize`` module and ``tokval`` is a string.
     """
     assert callable(f), 'f must be callable'
-    return tokenize.untokenize(lmap(f, tokenize_string(source)))
+    assert callable(g), 'g must be callable'
+    return tokenize.untokenize(lmap(f, tokenize_string(g(source))))
 
 
 def _is_type(t):
@@ -711,8 +736,10 @@ def visitor(x, y):
 class PandasExprVisitor(BaseExprVisitor):
 
     def __init__(self, env, engine, parser,
-                 preparser=partial(_preparse, f=_compose(_replace_locals,
-                                                         _replace_booleans))):
+                 preparser=partial(_preparse,
+                                   f=_compose(_replace_locals,
+                                              _replace_booleans),
+                                   g=_replace_spaces_backtickvariables)):
         super(PandasExprVisitor, self).__init__(env, engine, parser, preparser)
 
 

pandas/core/frame.py

@@ -3186,7 +3186,12 @@ def eval(self, expr, inplace=False, **kwargs):
         kwargs['level'] = kwargs.pop('level', 0) + 1
         if resolvers is None:
             index_resolvers = self._get_index_resolvers()
-            resolvers = dict(self.iteritems()), index_resolvers
+            # Alter names with spaces so that they can be found by backtick
+            # quoting. see also pandas/core/computation/expr.py
+            # _replace_spaces_backtickvariables(source)
+            column_resolvers = {com.clean_column_name_with_spaces(k): v
+                                for k, v in self.iteritems()}
+            resolvers = column_resolvers, index_resolvers
         if 'target' not in kwargs:
             kwargs['target'] = self
         kwargs['resolvers'] = kwargs.get('resolvers', ()) + tuple(resolvers)