ENH: backport Python 3.3 ChainMap

ChainMap implements a list of mappings that effectively functions as a single
dictionary. This class is very useful for implementing scope. This commit also
adds a DeepChainMap subclass of ChainMap for writing and deleting keys.

Author: cpcloud

pandas/compat/__init__.py

@@ -54,6 +54,8 @@
     import pickle as cPickle
     import http.client as httplib
 
+from chainmap import DeepChainMap
+
 
 if PY3:
     def isidentifier(s):

pandas/compat/chainmap.py

@@ -0,0 +1,20 @@
+try:
+    from collections import ChainMap
+except ImportError:
+    from chainmap_impl import ChainMap
+
+
+class DeepChainMap(ChainMap):
+    def __setitem__(self, key, value):
+        for mapping in self.maps:
+            if key in mapping:
+                mapping[key] = value
+                return
+        self.maps[0][key] = value
+
+    def __delitem__(self, key):
+        for mapping in self.maps:
+            if key in mapping:
+                del mapping[key]
+                return
+        raise KeyError(key)

pandas/compat/chainmap_impl.py

@@ -0,0 +1,132 @@
+from collections import MutableMapping
+from thread import get_ident
+
+
+def recursive_repr(fillvalue='...'):
+    'Decorator to make a repr function return fillvalue for a recursive call'
+
+    def decorating_function(user_function):
+        repr_running = set()
+
+        def wrapper(self):
+            key = id(self), get_ident()
+            if key in repr_running:
+                return fillvalue
+            repr_running.add(key)
+            try:
+                result = user_function(self)
+            finally:
+                repr_running.discard(key)
+            return result
+
+        # Can't use functools.wraps() here because of bootstrap issues
+        wrapper.__module__ = getattr(user_function, '__module__')
+        wrapper.__doc__ = getattr(user_function, '__doc__')
+        wrapper.__name__ = getattr(user_function, '__name__')
+        return wrapper
+
+    return decorating_function
+
+
+class ChainMap(MutableMapping):
+    ''' A ChainMap groups multiple dicts (or other mappings) together
+    to create a single, updateable view.
+
+    The underlying mappings are stored in a list.  That list is public and can
+    accessed or updated using the *maps* attribute.  There is no other state.
+
+    Lookups search the underlying mappings successively until a key is found.
+    In contrast, writes, updates, and deletions only operate on the first
+    mapping.
+
+    '''
+
+    def __init__(self, *maps):
+        '''Initialize a ChainMap by setting *maps* to the given mappings.
+        If no mappings are provided, a single empty dictionary is used.
+
+        '''
+        self.maps = list(maps) or [{}]          # always at least one map
+
+    def __missing__(self, key):
+        raise KeyError(key)
+
+    def __getitem__(self, key):
+        for mapping in self.maps:
+            try:
+                return mapping[key]             # can't use 'key in mapping' with defaultdict
+            except KeyError:
+                pass
+        return self.__missing__(key)            # support subclasses that define __missing__
+
+    def get(self, key, default=None):
+        return self[key] if key in self else default
+
+    def __len__(self):
+        return len(set().union(*self.maps))     # reuses stored hash values if possible
+
+    def __iter__(self):
+        return iter(set().union(*self.maps))
+
+    def __contains__(self, key):
+        return any(key in m for m in self.maps)
+
+    def __bool__(self):
+        return any(self.maps)
+
+    @recursive_repr()
+    def __repr__(self):
+        return '{0.__class__.__name__}({1})'.format(
+            self, ', '.join(repr(m) for m in self.maps))
+
+    @classmethod
+    def fromkeys(cls, iterable, *args):
+        'Create a ChainMap with a single dict created from the iterable.'
+        return cls(dict.fromkeys(iterable, *args))
+
+    def copy(self):
+        'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
+        return self.__class__(self.maps[0].copy(), *self.maps[1:])
+
+    __copy__ = copy
+
+    def new_child(self, m=None):                # like Django's Context.push()
+        '''
+        New ChainMap with a new map followed by all previous maps. If no
+        map is provided, an empty dict is used.
+        '''
+        if m is None:
+            m = {}
+        return self.__class__(m, *self.maps)
+
+    @property
+    def parents(self):                          # like Django's Context.pop()
+        'New ChainMap from maps[1:].'
+        return self.__class__(*self.maps[1:])
+
+    def __setitem__(self, key, value):
+        self.maps[0][key] = value
+
+    def __delitem__(self, key):
+        try:
+            del self.maps[0][key]
+        except KeyError:
+            raise KeyError('Key not found in the first mapping: {!r}'.format(key))
+
+    def popitem(self):
+        'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'
+        try:
+            return self.maps[0].popitem()
+        except KeyError:
+            raise KeyError('No keys found in the first mapping.')
+
+    def pop(self, key, *args):
+        'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'
+        try:
+            return self.maps[0].pop(key, *args)
+        except KeyError:
+            raise KeyError('Key not found in the first mapping: {!r}'.format(key))
+
+    def clear(self):
+        'Clear maps[0], leaving maps[1:] intact.'
+        self.maps[0].clear()

pandas/computation/engines.py

@@ -4,6 +4,7 @@
 import abc
 
 from pandas import compat
+from pandas.compat import DeepChainMap
 from pandas.core import common as com
 from pandas.computation.align import _align, _reconstruct_object
 from pandas.computation.ops import UndefinedVariableError
@@ -29,9 +30,6 @@ def convert(self):
         """
         return com.pprint_thing(self.expr)
 
-    def pre_evaluate(self):
-        self.expr.check_name_clashes()
-
     def evaluate(self):
         """Run the engine on the expression
 
@@ -47,7 +45,6 @@ def evaluate(self):
             self.result_type, self.aligned_axes = _align(self.expr.terms)
 
         # make sure no names in resolvers and locals/globals clash
-        self.pre_evaluate()
         res = self._evaluate()
         return _reconstruct_object(self.result_type, res, self.aligned_axes,
                                    self.expr.terms.return_type)
@@ -87,16 +84,14 @@ def convert(self):
     def _evaluate(self):
         import numexpr as ne
 
-        # add the resolvers to locals
-        self.expr.add_resolvers_to_locals()
-
         # convert the expression to a valid numexpr expression
         s = self.convert()
 
         try:
-            return ne.evaluate(s, local_dict=self.expr.env.locals,
-                               global_dict=self.expr.env.globals,
-                               truediv=self.expr.truediv)
+            env = self.expr.env
+            full_scope = DeepChainMap(*(env.resolvers.maps + env.scope.maps))
+            return ne.evaluate(s, local_dict=full_scope,
+                               truediv=env.scope['truediv'])
         except KeyError as e:
             # python 3 compat kludge
             try:
@@ -118,7 +113,6 @@ def __init__(self, expr):
         super(PythonEngine, self).__init__(expr)
 
     def evaluate(self):
-        self.pre_evaluate()
         return self.expr()
 
     def _evaluate(self):

pandas/computation/expr.py

@@ -631,14 +631,15 @@ def visit_Assign(self, node, **kwargs):
 
         try:
             assigner = self.visit(node.targets[0], **kwargs)
-        except UndefinedVariableError:
+        except (UndefinedVariableError, KeyError):
             assigner = node.targets[0].id
 
         self.assigner = getattr(assigner, 'name', assigner)
         if self.assigner is None:
             raise SyntaxError('left hand side of an assignment must be a '
                               'single resolvable name')
 
+        import ipdb; ipdb.set_trace()
         return self.visit(node.value, **kwargs)
 
     def visit_Attribute(self, node, **kwargs):