BUG: Correct out-of-bounds error with large indeces

closes pandas-dev#12527

Author: Gábor Lipták <[email protected]>

Closes pandas-dev#12921 from gliptak/keyerror1 and squashes the following commits:

02ca451 [Gábor Lipták] Correct out-of-bounds error with large indeces

Author: gliptak

doc/source/whatsnew/v0.18.1.txt

@@ -286,7 +286,7 @@ Bug Fixes
 - Bug in ``.quantile()`` with empty Series may return scalar rather than empty Series (:issue:`12772`)
 
 
-
+- Bug in ``.loc`` with out-of-bounds in a large indexer would raise ``IndexError`` rather than ``KeyError`` (:issue:`12527`)
 - Bug in equality testing with a ``Categorical`` in a ``DataFrame`` (:issue:`12564`)
 - Bug in ``GroupBy.first()``, ``.last()`` returns incorrect row when ``TimeGrouper`` is used (:issue:`7453`)
 

pandas/index.pyx

@@ -143,6 +143,8 @@ cdef class IndexEngine:
                 return self._get_loc_duplicates(val)
             values = self._get_index_values()
             loc = _bin_search(values, val) # .searchsorted(val, side='left')
+            if loc >= len(values):
+                raise KeyError(val)
             if util.get_value_at(values, loc) != val:
                 raise KeyError(val)
             return loc

pandas/indexes/multi.py

@@ -769,7 +769,7 @@ def to_hierarchical(self, n_repeat, n_shuffle=1):
         levels = self.levels
         labels = [np.repeat(x, n_repeat) for x in self.labels]
         # Assumes that each label is divisible by n_shuffle
-        labels = [x.reshape(n_shuffle, -1).ravel(1) for x in labels]
+        labels = [x.reshape(n_shuffle, -1).ravel('F') for x in labels]
         names = self.names
         return MultiIndex(levels=levels, labels=labels, names=names)
 

pandas/tests/indexes/test_multi.py

@@ -2052,6 +2052,23 @@ def test_equals_operator(self):
         # GH9785
         self.assertTrue((self.index == self.index).all())
 
+    def test_large_multiindex_error(self):
+        # GH12527
+        df_below_1000000 = pd.DataFrame(
+            1, index=pd.MultiIndex.from_product([[1, 2], range(499999)]),
+            columns=['dest'])
+        with assertRaises(KeyError):
+            df_below_1000000.loc[(-1, 0), 'dest']
+        with assertRaises(KeyError):
+            df_below_1000000.loc[(3, 0), 'dest']
+        df_above_1000000 = pd.DataFrame(
+            1, index=pd.MultiIndex.from_product([[1, 2], range(500001)]),
+            columns=['dest'])
+        with assertRaises(KeyError):
+            df_above_1000000.loc[(-1, 0), 'dest']
+        with assertRaises(KeyError):
+            df_above_1000000.loc[(3, 0), 'dest']
+
     def test_partial_string_timestamp_multiindex(self):
         # GH10331
         dr = pd.date_range('2016-01-01', '2016-01-03', freq='12H')