ENH: deep internal tinkering, cythonized cache_readonly, engines cython ext

Author: wesm

pandas/core/daterange.py

@@ -22,6 +22,8 @@ def f(self, other):
 _CACHE_START = datetime(1950, 1, 1)
 _CACHE_END   = datetime(2030, 1, 1)
 
+_daterange_cache = {}
+
 class DateRange(Index):
     """
     Fixed frequency date range according to input parameters.
@@ -43,7 +45,6 @@ class DateRange(Index):
     tzinfo : pytz.timezone
         To endow DateRange with time zone information
     """
-    _cache = {}
     def __new__(cls, start=None, end=None, periods=None,
                 offset=datetools.bday, time_rule=None,
                 tzinfo=None, name=None, **kwds):
@@ -143,17 +144,17 @@ def _cached_range(cls, start=None, end=None, periods=None, offset=None,
         if offset is None:
             raise Exception('Must provide a DateOffset!')
 
-        if offset not in cls._cache:
+        if offset not in _daterange_cache:
             xdr = generate_range(_CACHE_START, _CACHE_END, offset=offset)
             arr = np.array(list(xdr), dtype=object, copy=False)
 
             cachedRange = arr.view(DateRange)
             cachedRange.offset = offset
             cachedRange.tzinfo = None
             cachedRange.name = None
-            cls._cache[offset] = cachedRange
+            _daterange_cache[offset] = cachedRange
         else:
-            cachedRange = cls._cache[offset]
+            cachedRange = _daterange_cache[offset]
 
         if start is None:
             if end is None:
@@ -165,13 +166,13 @@ def _cached_range(cls, start=None, end=None, periods=None, offset=None,
 
             end = offset.rollback(end)
 
-            endLoc = cachedRange.indexMap[end] + 1
+            endLoc = cachedRange.get_loc(end) + 1
             startLoc = endLoc - periods
         elif end is None:
             assert(isinstance(start, datetime))
             start = offset.rollforward(start)
 
-            startLoc = cachedRange.indexMap[start]
+            startLoc = cachedRange.get_loc(start)
             if periods is None:
                 raise Exception('Must provide number of periods!')
 
@@ -180,8 +181,8 @@ def _cached_range(cls, start=None, end=None, periods=None, offset=None,
             start = offset.rollforward(start)
             end = offset.rollback(end)
 
-            startLoc = cachedRange.indexMap[start]
-            endLoc = cachedRange.indexMap[end] + 1
+            startLoc = cachedRange.get_loc(start)
+            endLoc = cachedRange.get_loc(end) + 1
 
         indexSlice = cachedRange[startLoc:endLoc]
         indexSlice.name = name

pandas/core/frame.py

@@ -865,7 +865,22 @@ def __array_wrap__(self, result):
                                  columns=self.columns, copy=False)
 
     #----------------------------------------------------------------------
-    # getitem/setitem related
+    # Getting and setting elements
+
+    def get(self, index, col):
+        """
+        Retrieve single value at passed column and index
+
+        Parameters
+        ----------
+        index : row label
+        col : column label
+
+        Returns
+        -------
+        element : scalar value
+        """
+        return self._data.get_scalar((col, index))
 
     def __getitem__(self, key):
         # slice rows

pandas/core/index.py

@@ -9,6 +9,7 @@
                                 _is_bool_indexer, _asarray_tuplesafe)
 from pandas.util.decorators import cache_readonly
 import pandas._tseries as lib
+import pandas._engines as _engines
 
 __all__ = ['Index']
 
@@ -93,19 +94,15 @@ def values(self):
     def is_monotonic(self):
         return self._is_monotonic(self.values)
 
-    _indexMap = None
-    _integrity = False
-
     @property
     def indexMap(self):
         "{label -> location}"
-        if self._indexMap is None:
-            self._indexMap = self._map_indices(self)
-            self._integrity = len(self._indexMap) == len(self)
+        return self._engine.get_mapping(1)
 
-        if not self._integrity:
-            raise Exception('Index cannot contain duplicate values!')
-        return self._indexMap
+    @cache_readonly
+    def _engine(self):
+        # property, for now, slow to look up
+        return _engines.DictIndexEngine(self.values, self._map_indices)
 
     def _get_level_number(self, level):
         if not isinstance(level, int):
@@ -114,12 +111,7 @@ def _get_level_number(self, level):
         return level
 
     def _verify_integrity(self):
-        if self._indexMap is None:
-            try:
-                self.indexMap
-            except Exception:
-                return False
-        return len(self.indexMap) == len(self)
+        return self._engine.has_integrity
 
     def _get_duplicates(self):
         from collections import defaultdict
@@ -139,7 +131,7 @@ def is_all_dates(self):
         return self._allDates
 
     def __iter__(self):
-        return iter(self.view(np.ndarray))
+        return iter(self.values)
 
     def __setstate__(self, state):
         """Necessary for making this object picklable"""
@@ -152,7 +144,7 @@ def __deepcopy__(self, memo={}):
         return self
 
     def __contains__(self, key):
-        return key in self.indexMap
+        return key in self._engine
 
     def __hash__(self):
         return hash(self.view(np.ndarray))
@@ -238,7 +230,7 @@ def asof(self, label):
         For a sorted index, return the most recent label up to and including
         the passed label. Return NaN if not found
         """
-        if label not in self.indexMap:
+        if label not in self:
             loc = self.searchsorted(label, side='left')
             if loc > 0:
                 return self[loc-1]
@@ -409,7 +401,7 @@ def get_loc(self, key):
         -------
         loc : int
         """
-        return self.indexMap[key]
+        return self._engine.get_loc(key)
 
     def get_indexer(self, target, method=None):
         """
@@ -570,14 +562,14 @@ def slice_locs(self, start=None, end=None):
         if start is None:
             beg_slice = 0
         elif start in self:
-            beg_slice = self.indexMap[start]
+            beg_slice = self.get_loc(start)
         else:
             beg_slice = self.searchsorted(start, side='left')
 
         if end is None:
             end_slice = len(self)
-        elif end in self.indexMap:
-            end_slice = self.indexMap[end] + 1
+        elif end in self:
+            end_slice = self.get_loc(end) + 1
         else:
             end_slice = self.searchsorted(end, side='right')
 
@@ -893,11 +885,6 @@ def __init__(self, levels, labels, sortorder=None, names=None,
     def dtype(self):
         return np.dtype('O')
 
-    def __iter__(self):
-        values = [np.asarray(lev).take(lab)
-                  for lev, lab in zip(self.levels, self.labels)]
-        return izip(*values)
-
     def _get_level_number(self, level):
         if not isinstance(level, int):
             count = self.names.count(level)
@@ -912,9 +899,9 @@ def _get_level_number(self, level):
 
     @property
     def values(self):
-        result = np.empty(len(self), dtype=object)
-        result[:] = list(self)
-        return result
+        values = [np.asarray(lev).take(lab)
+                  for lev, lab in zip(self.levels, self.labels)]
+        return lib.fast_zip(values)
 
     def get_level_values(self, level):
         """
@@ -935,8 +922,7 @@ def get_level_values(self, level):
 
     def __contains__(self, key):
         try:
-            label_key = self._get_label_key(key)
-            return label_key in self.indexMap
+            return key in self.indexMap
         except Exception:
             return False
 
@@ -1033,18 +1019,6 @@ def from_tuples(cls, tuples, sortorder=None, names=None):
         return MultiIndex.from_arrays(arrays, sortorder=sortorder,
                                       names=names)
 
-    @property
-    def indexMap(self):
-        if self._indexMap is None:
-            zipped = zip(*self.labels)
-            self._indexMap = lib.map_indices_list(zipped)
-            self._integrity = len(self._indexMap) == len(self)
-
-        if not self._integrity:
-            raise Exception('Index cannot contain duplicate values!')
-
-        return self._indexMap
-
     @property
     def nlevels(self):
         return len(self.levels)
@@ -1399,7 +1373,7 @@ def get_loc(self, key):
         """
         if isinstance(key, tuple):
             if len(key) == self.nlevels:
-                return self._get_tuple_loc(key)
+                return self._engine.get_loc(key)
             else:
                 result = slice(*self.slice_locs(key, key))
                 if result.start == result.stop:
@@ -1418,13 +1392,6 @@ def get_loc(self, key):
                 j = labels.searchsorted(loc, side='right')
                 return slice(i, j)
 
-    def _get_tuple_loc(self, tup):
-        indexer = self._get_label_key(tup)
-        try:
-            return self.indexMap[indexer]
-        except KeyError:
-            raise KeyError(str(tup))
-
     def _get_label_key(self, tup):
         return tuple(lev.get_loc(v) for lev, v in zip(self.levels, tup))
 

pandas/core/internals.py

@@ -315,8 +315,7 @@ def shape(self):
         return tuple(len(ax) for ax in self.axes)
 
     def _verify_integrity(self):
-        union_items = _union_block_items(self.blocks)
-
+        _union_block_items(self.blocks)
         mgr_shape = self.shape
         for block in self.blocks:
             assert(block.values.shape[1:] == mgr_shape[1:])
@@ -520,6 +519,19 @@ def get(self, item):
         _, block = self._find_block(item)
         return block.get(item)
 
+    def get_scalar(self, tup):
+        """
+        Retrieve single item
+        """
+        item = tup[0]
+        _, blk = self._find_block(item)
+
+        # this could obviously be seriously sped up in cython
+        item_loc = blk.items.get_loc(item),
+        full_loc = item_loc + tuple(ax.get_loc(x)
+                                    for ax, x in zip(self.axes[1:], tup[1:]))
+        return blk.values[full_loc]
+
     def delete(self, item):
         i, _ = self._find_block(item)
         loc = self.items.get_loc(item)
@@ -548,6 +560,9 @@ def set(self, item, value):
             # insert at end
             self.insert(len(self.items), item, value)
 
+    def set_scalar(self, tup, value):
+        pass
+
     def insert(self, loc, item, value):
         if item in self.items:
             raise Exception('cannot insert %s, already exists' % item)
@@ -903,23 +918,15 @@ def _blocks_to_series_dict(blocks, index=None):
     return series_dict
 
 def _interleaved_dtype(blocks):
-    have_int = False
-    have_bool = False
-    have_object = False
-    have_float = False
-
-    for block in blocks:
-        if isinstance(block, FloatBlock):
-            have_float = True
-        elif isinstance(block, IntBlock):
-            have_int = True
-        elif isinstance(block, BoolBlock):
-            have_bool = True
-        elif isinstance(block, ObjectBlock):
-            have_object = True
-        else: # pragma: no cover
-            raise Exception('Unrecognized block type')
-
+    from collections import defaultdict
+    counts = defaultdict(lambda: 0)
+    for x in blocks:
+        counts[type(x)] += 1
+
+    have_int = counts[IntBlock] > 0
+    have_bool = counts[BoolBlock] > 0
+    have_object = counts[ObjectBlock] > 0
+    have_float = counts[FloatBlock] > 0
     have_numeric = have_float or have_int
 
     if have_object: