WIP: high memory in MI

xref pandas-dev#13904

Author: jreback

asv_bench/benchmarks/indexing.py

@@ -88,7 +88,7 @@ def setup(self):
 
     def time_getitem_scalar(self):
         self.ts[self.dt]
-    
+
 
 class DataFrameIndexing(object):
     goal_time = 0.2
@@ -189,6 +189,7 @@ def setup(self):
         self.eps_C = 5
         self.eps_D = 5000
         self.mdt2 = self.mdt.set_index(['A', 'B', 'C', 'D']).sortlevel()
+        self.miint = MultiIndex.from_product([np.arange(1000), np.arange(1000)], names=['one','two'])
 
     def time_series_xs_mi_ix(self):
         self.s.ix[999]
@@ -199,6 +200,10 @@ def time_frame_xs_mi_ix(self):
     def time_multiindex_slicers(self):
         self.mdt2.loc[self.idx[(self.test_A - self.eps_A):(self.test_A + self.eps_A), (self.test_B - self.eps_B):(self.test_B + self.eps_B), (self.test_C - self.eps_C):(self.test_C + self.eps_C), (self.test_D - self.eps_D):(self.test_D + self.eps_D)], :]
 
+    def time_multiindex_get_indexer(self):
+        self.miint.get_indexer(np.array([(0, 10), (0, 11), (0, 12), (0, 13), (0, 14),
+                                         (0, 15), (0, 16),(0, 17), (0, 18), (0, 19)], dtype=object))
+
 
 class PanelIndexing(object):
     goal_time = 0.2

pandas/core/algorithms.py

@@ -1250,7 +1250,7 @@ def take_nd(arr, indexer, axis=0, out=None, fill_value=np.nan, mask_info=None,
         indexer = np.arange(arr.shape[axis], dtype=np.int64)
         dtype, fill_value = arr.dtype, arr.dtype.type()
     else:
-        indexer = _ensure_int64(indexer)
+        indexer = _ensure_int64(indexer, copy=False)
         if not allow_fill:
             dtype, fill_value = arr.dtype, arr.dtype.type()
             mask_info = None, False
@@ -1303,7 +1303,6 @@ def take_nd(arr, indexer, axis=0, out=None, fill_value=np.nan, mask_info=None,
 
     func = _get_take_nd_function(arr.ndim, arr.dtype, out.dtype, axis=axis,
                                  mask_info=mask_info)
-    indexer = _ensure_int64(indexer)
     func(arr, indexer, out, fill_value)
 
     if flip_order:

pandas/hashtable.pxd

@@ -30,6 +30,14 @@ cdef class PyObjectHashTable(HashTable):
     cpdef get_item(self, object val)
     cpdef set_item(self, object key, Py_ssize_t val)
 
+cdef class MultiIndexHashTable(HashTable):
+    cdef:
+        kh_uint64_t *table
+        int64_t nlevels
+
+    cpdef get_item(self, object val)
+    cpdef set_item(self, object key, Py_ssize_t val)
+
 cdef class StringHashTable(HashTable):
     cdef kh_str_t *table
 

pandas/index.pyx

@@ -1,4 +1,4 @@
-# cython: profile=False
+# cython: profile=True
 
 from numpy cimport ndarray
 
@@ -275,7 +275,6 @@ cdef class IndexEngine:
         if not self.is_mapping_populated:
 
             values = self._get_index_values()
-
             self.mapping = self._make_hash_table(len(values))
             self.mapping.map_locations(values)
 
@@ -545,5 +544,77 @@ cdef inline bint _is_utc(object tz):
     return tz is UTC or isinstance(tz, _du_utc)
 
 
+cdef class MultiIndexEngine(IndexEngine):
+
+    def __init__(self, vgetter, n):
+        super(MultiIndexEngine, self).__init__(vgetter, n)
+
+        # by definition we *always* want to use our hashtable
+        # and not bin searching
+        self.over_size_threshold = False
+
+    def _call_monotonic(self, object mi):
+        cdef:
+            object values
+
+        values = mi._to_multiindex_structure()
+        return values.is_lexsorted, False, values.is_unique
+
+    def get_backfill_indexer(self, other, limit=None):
+        return algos.backfill_object(self._get_index_values(),
+                                        other, limit=limit)
+
+    def get_pad_indexer(self, other, limit=None):
+        return algos.pad_object(self._get_index_values(),
+                                   other, limit=limit)
+
+    cpdef get_loc(self, object val):
+        if is_definitely_invalid_key(val):
+            raise TypeError("'{val}' is an invalid key".format(val=val))
+
+        self._ensure_mapping_populated()
+        if not self.unique:
+            return self._get_loc_duplicates(val)
+
+        try:
+            return self.mapping.get_item(val)
+        except TypeError:
+            raise KeyError(val)
+
+    cdef _make_hash_table(self, n):
+        return _hash.MultiIndexHashTable(n)
+
+    cdef _maybe_get_bool_indexer(self, object val):
+        cdef:
+            ndarray[uint8_t, cast=True] indexer
+            ndarray[object] values
+            int count = 0
+            Py_ssize_t i, n
+            int last_true
+
+        if not util.is_integer_object(val):
+            raise KeyError(val)
+
+        values = self._get_index_values()
+        n = len(values)
+
+        result = np.empty(n, dtype=bool)
+        indexer = result.view(np.uint8)
+
+        for i in range(n):
+            if values[i] == val:
+                count += 1
+                indexer[i] = 1
+                last_true = i
+            else:
+                indexer[i] = 0
+
+        if count == 0:
+            raise KeyError(val)
+        if count == 1:
+            return last_true
+
+        return result
+
 # Generated from template.
 include "index_class_helper.pxi"

pandas/indexes/base.py

@@ -2404,7 +2404,6 @@ def _get_fill_indexer_searchsorted(self, target, method, limit=None):
                              'if index and target are monotonic' % method)
 
         side = 'left' if method == 'pad' else 'right'
-        target = np.asarray(target)
 
         # find exact matches first (this simplifies the algorithm)
         indexer = self.get_indexer(target)

pandas/indexes/multi.py

@@ -14,7 +14,6 @@
 from pandas.compat.numpy import function as nv
 from pandas import compat
 
-
 from pandas.types.common import (_ensure_int64,
                                  _ensure_platform_int,
                                  is_object_dtype,
@@ -73,6 +72,7 @@ class MultiIndex(Index):
     _levels = FrozenList()
     _labels = FrozenList()
     _comparables = ['names']
+    _engine_type = _index.MultiIndexEngine
     rename = Index.set_names
 
     def __new__(cls, levels=None, labels=None, sortorder=None, names=None,
@@ -114,7 +114,6 @@ def __new__(cls, levels=None, labels=None, sortorder=None, names=None,
             result._verify_integrity()
         if _set_identity:
             result._reset_identity()
-
         return result
 
     def _verify_integrity(self, labels=None, levels=None):
@@ -613,6 +612,38 @@ def _get_level_number(self, level):
 
     _tuples = None
 
+    def _to_multiindex_structure(self):
+
+        # return a structure that has an efficient representation
+        # of the hashed values of the multi-index
+        # as well as properties for interfacing with index.pyx
+        # and hashtable.pyx
+        from pandas.tools.hashing import hash_tuples
+
+        class _MultiIndexStructure(object):
+            def __init__(self, mi):
+                self.nlevels = mi.nlevels
+                self.length = len(mi)
+                self.obj = mi
+                self.values = hash_tuples(mi)
+
+            def __len__(self):
+                return self.length
+
+            @property
+            def is_lexsorted(self):
+                return self.obj.is_lexsorted()
+
+            @property
+            def is_unique(self):
+                return self.obj.is_unique
+
+        return _MultiIndexStructure(self)
+
+    @cache_readonly
+    def _engine(self):
+        return self._engine_type(lambda: self, len(self))
+
     @property
     def values(self):
         if self._tuples is not None:
@@ -846,7 +877,8 @@ def to_frame(self, index=True):
         from pandas import DataFrame
         result = DataFrame({(name or level): self.get_level_values(level)
                             for name, level in
-                            zip(self.names, range(len(self.levels)))})
+                            zip(self.names, range(len(self.levels)))},
+                           copy=False)
         if index:
             result.index = self
         return result
@@ -1472,29 +1504,23 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None):
         method = missing.clean_reindex_fill_method(method)
         target = _ensure_index(target)
 
-        target_index = target
-        if isinstance(target, MultiIndex):
-            target_index = target._tuple_index
-
-        if not is_object_dtype(target_index.dtype):
-            return np.ones(len(target_index)) * -1
+        if not isinstance(target, MultiIndex):
+            target = MultiIndex.from_tuples(target)
 
         if not self.is_unique:
             raise Exception('Reindexing only valid with uniquely valued Index '
                             'objects')
 
-        self_index = self._tuple_index
-
         if method == 'pad' or method == 'backfill':
             if tolerance is not None:
                 raise NotImplementedError("tolerance not implemented yet "
                                           'for MultiIndex')
-            indexer = self_index._get_fill_indexer(target, method, limit)
+            indexer = self._get_fill_indexer(target, method, limit)
         elif method == 'nearest':
             raise NotImplementedError("method='nearest' not implemented yet "
                                       'for MultiIndex; see GitHub issue 9365')
         else:
-            indexer = self_index._engine.get_indexer(target._values)
+            indexer = self._engine.get_indexer(target)
 
         return _ensure_platform_int(indexer)
 
@@ -1561,17 +1587,6 @@ def reindex(self, target, method=None, level=None, limit=None,
 
         return target, indexer
 
-    @cache_readonly
-    def _tuple_index(self):
-        """
-        Convert MultiIndex to an Index of tuples
-
-        Returns
-        -------
-        index : Index
-        """
-        return Index(self._values)
-
     def get_slice_bound(self, label, side, kind):
 
         if not isinstance(label, tuple):

pandas/io/pytables.py

@@ -3789,9 +3789,9 @@ def read(self, where=None, columns=None, **kwargs):
                 lp = DataFrame(c.data, index=long_index, columns=c.values)
 
                 # need a better algorithm
-                tuple_index = long_index._tuple_index
+                tuple_index = long_index.values
 
-                unique_tuples = lib.fast_unique(tuple_index.values)
+                unique_tuples = lib.fast_unique(tuple_index)
                 unique_tuples = _asarray_tuplesafe(unique_tuples)
 
                 indexer = match(unique_tuples, tuple_index)

pandas/src/algos_common_helper.pxi.in

@@ -567,12 +567,12 @@ def get_dispatch(dtypes):
 
 {{for name, c_type, dtype in get_dispatch(dtypes)}}
 
-cpdef ensure_{{name}}(object arr):
+cpdef ensure_{{name}}(object arr, copy=True):
     if util.is_array(arr):
         if (<ndarray> arr).descr.type_num == NPY_{{c_type}}:
             return arr
         else:
-            return arr.astype(np.{{dtype}})
+            return arr.astype(np.{{dtype}}, copy=copy)
     else:
         return np.array(arr, dtype=np.{{dtype}})