BUG: groupby does not respect dropna=False when input has MultiIndex (pandas-dev#47186)

Author: rhshadrach

doc/source/whatsnew/v1.5.0.rst

@@ -873,6 +873,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.rolling` gives ValueError when center=True, axis=1 and win_type is specified (:issue:`46135`)
 - Bug in :meth:`.DataFrameGroupBy.describe` and :meth:`.SeriesGroupBy.describe` produces inconsistent results for empty datasets (:issue:`41575`)
 - Bug in :meth:`DataFrame.resample` reduction methods when used with ``on`` would attempt to aggregate the provided column (:issue:`47079`)
+- Bug in :meth:`DataFrame.groupby` and :meth:`Series.groupby` would not respect ``dropna=False`` when the input DataFrame/Series had a NaN values in a :class:`MultiIndex` (:issue:`46783`)
 
 Reshaping
 ^^^^^^^^^

pandas/core/groupby/grouper.py

@@ -501,7 +501,7 @@ def __init__(
                 self.grouping_vector,  # Index
                 self._codes,
                 self._group_index,
-            ) = index._get_grouper_for_level(mapper, level=ilevel)
+            ) = index._get_grouper_for_level(mapper, level=ilevel, dropna=dropna)
 
         # a passed Grouper like, directly get the grouper in the same way
         # as single grouper groupby, use the group_info to get codes

pandas/core/indexes/base.py

@@ -2227,7 +2227,11 @@ def _drop_level_numbers(self, levnums: list[int]):
             )
 
     def _get_grouper_for_level(
-        self, mapper, *, level=None
+        self,
+        mapper,
+        *,
+        level=None,
+        dropna: bool = True,
     ) -> tuple[Index, npt.NDArray[np.signedinteger] | None, Index | None]:
         """
         Get index grouper corresponding to an index level
@@ -2238,6 +2242,8 @@ def _get_grouper_for_level(
             Function mapping index values to groups
         level : int or None
             Index level, positional
+        dropna : bool
+            dropna from groupby
 
         Returns
         -------

pandas/core/indexes/multi.py

@@ -1503,42 +1503,30 @@ def _set_names(self, names, *, level=None, validate: bool = True):
 
     @doc(Index._get_grouper_for_level)
     def _get_grouper_for_level(
-        self, mapper, *, level=None
+        self,
+        mapper,
+        *,
+        level=None,
+        dropna: bool = True,
     ) -> tuple[Index, npt.NDArray[np.signedinteger] | None, Index | None]:
-        indexer = self.codes[level]
-        level_index = self.levels[level]
-
         if mapper is not None:
+            indexer = self.codes[level]
             # Handle group mapping function and return
             level_values = self.levels[level].take(indexer)
             grouper = level_values.map(mapper)
             return grouper, None, None
 
-        codes, uniques = algos.factorize(indexer, sort=True)
-
-        if len(uniques) > 0 and uniques[0] == -1:
-            # Handle NAs
-            mask = indexer != -1
-            ok_codes, uniques = algos.factorize(indexer[mask], sort=True)
-
-            codes = np.empty(len(indexer), dtype=indexer.dtype)
-            codes[mask] = ok_codes
-            codes[~mask] = -1
-
-        if len(uniques) < len(level_index):
-            # Remove unobserved levels from level_index
-            level_index = level_index.take(uniques)
-        else:
-            # break references back to us so that setting the name
-            # on the output of a groupby doesn't reflect back here.
-            level_index = level_index.copy()
+        values = self.get_level_values(level)
+        na_sentinel = -1 if dropna else None
+        codes, uniques = algos.factorize(values, sort=True, na_sentinel=na_sentinel)
+        assert isinstance(uniques, Index)
 
-        if level_index._can_hold_na:
-            grouper = level_index.take(codes, fill_value=True)
+        if self.levels[level]._can_hold_na:
+            grouper = uniques.take(codes, fill_value=True)
         else:
-            grouper = level_index.take(codes)
+            grouper = uniques.take(codes)
 
-        return grouper, codes, level_index
+        return grouper, codes, uniques
 
     @cache_readonly
     def inferred_type(self) -> str:

pandas/tests/groupby/test_groupby_dropna.py

@@ -332,6 +332,36 @@ def test_groupby_apply_with_dropna_for_multi_index(dropna, data, selected_data,
     tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.parametrize("input_index", [None, ["a"], ["a", "b"]])
+@pytest.mark.parametrize("keys", [["a"], ["a", "b"]])
+@pytest.mark.parametrize("series", [True, False])
+def test_groupby_dropna_with_multiindex_input(input_index, keys, series):
+    # GH#46783
+    obj = pd.DataFrame(
+        {
+            "a": [1, np.nan],
+            "b": [1, 1],
+            "c": [2, 3],
+        }
+    )
+
+    expected = obj.set_index(keys)
+    if series:
+        expected = expected["c"]
+    elif input_index == ["a", "b"] and keys == ["a"]:
+        # Column b should not be aggregated
+        expected = expected[["c"]]
+
+    if input_index is not None:
+        obj = obj.set_index(input_index)
+    gb = obj.groupby(keys, dropna=False)
+    if series:
+        gb = gb["c"]
+    result = gb.sum()
+
+    tm.assert_equal(result, expected)
+
+
 def test_groupby_nan_included():
     # GH 35646
     data = {"group": ["g1", np.nan, "g1", "g2", np.nan], "B": [0, 1, 2, 3, 4]}

pandas/tests/groupby/test_grouping.py

@@ -471,7 +471,7 @@ def test_multiindex_negative_level(self, mframe):
         tm.assert_frame_equal(result, expected)
 
         result = mframe.groupby(level=[-2, -1]).sum()
-        expected = mframe
+        expected = mframe.sort_index()
         tm.assert_frame_equal(result, expected)
 
         result = mframe.groupby(level=[-1, "first"]).sum()