ENH: pd.MultiIndex.get_loc(np.nan) (pandas-dev#19132)

MultiIndex.get_loc could not find nan with values including missing
values as a input

Author: proost

doc/source/whatsnew/v1.0.0.rst

@@ -229,6 +229,7 @@ Indexing
 - Bug in reindexing a :meth:`PeriodIndex` with another type of index that contained a `Period` (:issue:`28323`) (:issue:`28337`)
 - Fix assignment of column via `.loc` with numpy non-ns datetime type (:issue:`27395`)
 - Bug in :meth:`Float64Index.astype` where ``np.inf`` was not handled properly when casting to an integer dtype (:issue:`28475`)
+- When index is ``MultiIndex``, Using ``.get_loc`` can't find ``nan`` with values including missing values as input (:issue:`19132`)
 
 Missing
 ^^^^^^^

pandas/core/indexes/multi.py

@@ -2591,7 +2591,7 @@ def _partial_tup_index(self, tup, side="left"):
         for k, (lab, lev, labs) in enumerate(zipped):
             section = labs[start:end]
 
-            if lab not in lev:
+            if lab not in lev and not isna(lab):
                 if not lev.is_type_compatible(lib.infer_dtype([lab], skipna=False)):
                     raise TypeError("Level type mismatch: %s" % lab)
 
@@ -2601,13 +2601,38 @@ def _partial_tup_index(self, tup, side="left"):
                     loc -= 1
                 return start + section.searchsorted(loc, side=side)
 
-            idx = lev.get_loc(lab)
+            idx = self._get_loc_single_level_index(lev, lab)
             if k < n - 1:
                 end = start + section.searchsorted(idx, side="right")
                 start = start + section.searchsorted(idx, side="left")
             else:
                 return start + section.searchsorted(idx, side=side)
 
+    def _get_loc_single_level_index(self, level_index: Index, key) -> int:
+        """
+        If key is NA value, location of index unify as -1.
+
+        Parameters
+        ----------
+        level_index: Index
+        key : label
+
+        Returns
+        -------
+        loc : int
+            If key is NA value, loc is -1
+            Else, location of key in index
+
+        See Also
+        --------
+        Index.get_loc : The get_loc method for (single-level) index.
+        """
+
+        if is_scalar(key) and isna(key):
+            return -1
+        else:
+            return level_index.get_loc(key)
+
     def get_loc(self, key, method=None):
         """
         Get location for a label or a tuple of labels as an integer, slice or
@@ -2707,7 +2732,9 @@ def _maybe_to_slice(loc):
         loc = np.arange(start, stop, dtype="int64")
 
         for i, k in enumerate(follow_key, len(lead_key)):
-            mask = self.codes[i][loc] == self.levels[i].get_loc(k)
+            mask = self.codes[i][loc] == self._get_loc_single_level_index(
+                self.levels[i], k
+            )
             if not mask.all():
                 loc = loc[mask]
             if not len(loc):
@@ -2934,8 +2961,7 @@ def convert_indexer(start, stop, step, indexer=indexer, codes=level_codes):
                 return slice(i, j, step)
 
         else:
-
-            code = level_index.get_loc(key)
+            code = self._get_loc_single_level_index(level_index, key)
 
             if level > 0 or self.lexsort_depth == 0:
                 # Desired level is not sorted

pandas/tests/indexes/multi/test_contains.py

@@ -98,3 +98,13 @@ def test_isin_level_kwarg():
 
     with pytest.raises(KeyError, match="'Level C not found'"):
         idx.isin(vals_1, level="C")
+
+
+def test_contains_with_missing_value():
+    # issue 19132
+    idx = MultiIndex.from_arrays([[1, np.nan, 2]])
+    assert np.nan in idx
+
+    idx = MultiIndex.from_arrays([[1, 2], [np.nan, 3]])
+    assert np.nan not in idx
+    assert (1, np.nan) in idx

pandas/tests/indexes/multi/test_indexing.py

@@ -439,3 +439,82 @@ def test_timestamp_multiindex_indexer():
     )
     should_be = pd.Series(data=np.arange(24, len(qidx) + 24), index=qidx, name="foo")
     tm.assert_series_equal(result, should_be)
+
+
+def test_get_loc_with_values_including_missing_values():
+    # issue 19132
+    idx = MultiIndex.from_product([[np.nan, 1]] * 2)
+    expected = slice(0, 2, None)
+    assert idx.get_loc(np.nan) == expected
+
+    idx = MultiIndex.from_arrays([[np.nan, 1, 2, np.nan]])
+    expected = np.array([True, False, False, True])
+    tm.assert_numpy_array_equal(idx.get_loc(np.nan), expected)
+
+    idx = MultiIndex.from_product([[np.nan, 1]] * 3)
+    expected = slice(2, 4, None)
+    assert idx.get_loc((np.nan, 1)) == expected
+
+
+def test_get_indexer_with_missing_value():
+    # issue 19132
+    idx = MultiIndex.from_arrays([[1, np.nan, 2], [3, 4, 5]])
+    result = idx.get_indexer([1, np.nan, 2])
+    expected = np.array([-1, -1, -1], dtype="int64")
+    tm.assert_numpy_array_equal(result.astype("int64"), expected)
+
+    result = idx.get_indexer([(np.nan, 4)])
+    expected = np.array([1], dtype="int64")
+    tm.assert_numpy_array_equal(result.astype("int64"), expected)
+
+
+def test_get_slice_bound_with_missing_value():
+    # issue 19132
+    idx = MultiIndex.from_arrays([[np.nan, "a", "b"], ["c", "d", "e"]])
+    result = idx.get_slice_bound(np.nan, side="left", kind="ix")
+    expected = 0
+    assert result == expected
+
+    result = idx.get_slice_bound(np.nan, side="right", kind="ix")
+    expected = 1
+    assert result == expected
+
+    result = idx.get_slice_bound((np.nan, "c"), side="left", kind="ix")
+    expected = 0
+    assert result == expected
+
+    result = idx.get_slice_bound((np.nan, "c"), side="right", kind="ix")
+    expected = 1
+    assert result == expected
+
+
+def test_slice_indexer_with_missing_value():
+    # issue 19132
+    idx = MultiIndex.from_arrays([[np.nan, 1, 2], [3, 4, 5]])
+    result = idx.slice_indexer(start=np.nan, end=1)
+    expected = slice(0, 2, None)
+    assert result == expected
+
+    result = idx.slice_indexer(start=(np.nan, 3), end=(1, 4))
+    expected = slice(0, 2, None)
+    assert result == expected
+
+    result = idx.slice_indexer(start=3, end=4)
+    expected = slice(3, 3, None)
+    assert result == expected
+
+
+def test_slice_locs_with_missing_value():
+    # issue 19132
+    idx = MultiIndex.from_arrays([[np.nan, "a", "a", "b"], ["c", "d", "e", "e"]])
+    result = idx.slice_locs(start=np.nan)
+    expected = (0, 4)
+    assert result == expected
+
+    result = idx.slice_locs(start=np.nan, end="a")
+    expected = (0, 3)
+    assert result == expected
+
+    result = idx.slice_locs(start=np.nan, end=("a", "e"))
+    expected = (0, 3)
+    assert result == expected