BUG: MultiIndex indexing with >= 1000000 elements close #1757

Author: wesm

RELEASE.rst

@@ -89,6 +89,8 @@ pandas 0.8.2
   - Fix handling of general objects in isnull on which bool(...) fails (#1749)
   - Fix .ix indexing with MultiIndex ambiguity (#1678)
   - Fix .ix setting logic error with non-unique MultiIndex (#1750)
+  - Basic indexing now works on MultiIndex with > 1000000 elements, regression
+    from earlier version of pandas (#1757)
 
 pandas 0.8.1
 ============

pandas/src/engines.pyx

@@ -46,7 +46,7 @@ def set_value_at(ndarray arr, object loc, object val):
 
 
 # Don't populate hash tables in monotonic indexes larger than this
-cdef int _SIZE_CUTOFF = 1000000
+_SIZE_CUTOFF = 1000000
 
 
 cdef class IndexEngine:
@@ -116,7 +116,7 @@ cdef class IndexEngine:
             if not self.is_unique:
                 return self._get_loc_duplicates(val)
             values = self._get_index_values()
-            loc = values.searchsorted(val, side='left')
+            loc = _bin_search(values, val) # .searchsorted(val, side='left')
             if util.get_value_at(values, loc) != val:
                 raise KeyError(val)
             return loc
@@ -331,6 +331,32 @@ cdef class Float64Engine(IndexEngine):
         return _algos.backfill_float64(self._get_index_values(), other,
                                          limit=limit)
 
+
+cdef Py_ssize_t _bin_search(ndarray values, object val):
+    cdef:
+        Py_ssize_t mid, lo = 0, hi = len(values) - 1
+        object pval
+
+    if hi >= 0 and val > util.get_value_at(values, hi):
+        return len(values)
+
+    while lo < hi:
+        mid = (lo + hi) // 2
+        pval = util.get_value_at(values, mid)
+        if val < pval:
+            hi = mid
+        elif val > pval:
+            lo = mid + 1
+        else:
+            while mid > 0 and val == util.get_value_at(values, mid - 1):
+                mid -= 1
+            return mid
+
+    if val <= util.get_value_at(values, mid):
+        return mid
+    else:
+        return mid + 1
+
 _pad_functions = {
     'object' : _algos.pad_object,
     'int64' : _algos.pad_int64,

pandas/tests/test_multilevel.py

@@ -1467,6 +1467,23 @@ def test_nonunique_assignment_1750(self):
 
         self.assert_((df.xs((1, 1))['C'] == '_').all())
 
+    def test_indexing_over_hashtable_size_cutoff(self):
+        n = 10000
+
+        import pandas.lib as lib
+        old_cutoff = lib._SIZE_CUTOFF
+        lib._SIZE_CUTOFF = 20000
+
+        s = Series(np.arange(n),
+                   MultiIndex.from_arrays((["a"] * n, np.arange(n))))
+
+        # hai it works!
+        self.assertEquals(s[("a", 5)], 5)
+        self.assertEquals(s[("a", 6)], 6)
+        self.assertEquals(s[("a", 7)], 7)
+
+        lib._SIZE_CUTOFF = old_cutoff
+
 if __name__ == '__main__':
 
     # unittest.main()