BUG: DataFrame.groupby(., dropna=True, axis=0) incorrectly throws ShapeError

doc/source/whatsnew/v1.2.0.rst

@@ -221,7 +221,7 @@ Indexing
 Missing
 ^^^^^^^
 
-- Bug in :meth:`SeriesGroupBy.transform` now correctly handles missing values for `dropna=False` (:issue:`35014`)
+- Bug in :meth:`SeriesGroupBy.transform` and :meth:`DataFrameGroupBy.transform` now correctly handle missing values (:issue:`35014` and :issue:`35612`)
 -
 
 MultiIndex

pandas/core/groupby/generic.py

@@ -556,8 +556,9 @@ def _transform_general(
             if common_dtype is result.dtype:
                 result = maybe_downcast_numeric(result, self._selected_obj.dtype)
 
-        result.name = self._selected_obj.name
-        result.index = self._selected_obj.index
+        obj = self._selected_obj.dropna() if self.dropna else self._selected_obj
+        result.name = obj.name
+        result.index = obj.index
         return result
 
     def _transform_fast(self, result) -> Series:
@@ -1677,12 +1678,17 @@ def _gotitem(self, key, ndim: int, subset=None):
                 exclusions=self.exclusions,
                 as_index=self.as_index,
                 observed=self.observed,
+                dropna=self.dropna,
             )
         elif ndim == 1:
             if subset is None:
                 subset = self.obj[key]
             return SeriesGroupBy(
-                subset, selection=key, grouper=self.grouper, observed=self.observed
+                subset,
+                selection=key,
+                grouper=self.grouper,
+                observed=self.observed,
+                dropna=self.dropna,
             )
 
         raise AssertionError("invalid ndim for _gotitem")

pandas/core/groupby/groupby.py

@@ -690,7 +690,8 @@ def _set_result_index_ordered(self, result):
             result.set_axis(index, axis=self.axis, inplace=True)
             result = result.sort_index(axis=self.axis)
 
-        result.set_axis(self.obj._get_axis(self.axis), axis=self.axis, inplace=True)
+        obj = self.obj.dropna() if self.dropna else self.obj
+        result.set_axis(obj._get_axis(self.axis), axis=self.axis, inplace=True)
         return result
 
     def _dir_additions(self):

pandas/tests/groupby/test_groupby_dropna.py

@@ -165,12 +165,7 @@ def test_groupby_dropna_series_by(dropna, expected):
 @pytest.mark.parametrize(
     "dropna,df_expected,s_expected",
     [
-        pytest.param(
-            True,
-            pd.DataFrame({"B": [2, 2, 1]}),
-            pd.Series(data=[2, 2, 1], name="B"),
-            marks=pytest.mark.xfail(raises=ValueError),
-        ),
+        (True, pd.DataFrame({"B": [2, 2, 1]}), pd.Series(data=[2, 2, 1], name="B"),),
         (
             False,
             pd.DataFrame({"B": [2, 2, 1, 1]}),
@@ -179,7 +174,7 @@ def test_groupby_dropna_series_by(dropna, expected):
     ],
 )
 def test_slice_groupby_then_transform(dropna, df_expected, s_expected):
-    # GH35014
+    # GH35014 & GH35612
 
     df = pd.DataFrame({"A": [0, 0, 1, None], "B": [1, 2, 3, None]})
     gb = df.groupby("A", dropna=dropna)

pandas/tests/groupby/transform/test_transform.py

@@ -309,11 +309,11 @@ def test_transform_multiple(ts):
 def test_dispatch_transform(tsframe):
     df = tsframe[::5].reindex(tsframe.index)
 
-    grouped = df.groupby(lambda x: x.month)
+    grouped = df.groupby(lambda x: x.month, dropna=False)
 
     filled = grouped.fillna(method="pad")
     fillit = lambda x: x.fillna(method="pad")
-    expected = df.groupby(lambda x: x.month).transform(fillit)
+    expected = df.groupby(lambda x: x.month, dropna=False).transform(fillit)
     tm.assert_frame_equal(filled, expected)
 
 
@@ -412,10 +412,10 @@ def nsum(x):
         return np.nansum(x)
 
     results = [
-        df.groupby("col1").transform(sum)["col2"],
-        df.groupby("col1")["col2"].transform(sum),
-        df.groupby("col1").transform(nsum)["col2"],
-        df.groupby("col1")["col2"].transform(nsum),
+        df.groupby("col1", dropna=False).transform(sum)["col2"],
+        df.groupby("col1", dropna=False)["col2"].transform(sum),
+        df.groupby("col1", dropna=False).transform(nsum)["col2"],
+        df.groupby("col1", dropna=False)["col2"].transform(nsum),
     ]
     for result in results:
         tm.assert_series_equal(result, expected, check_names=False)
@@ -448,7 +448,9 @@ def test_groupby_transform_with_int():
         )
     )
     with np.errstate(all="ignore"):
-        result = df.groupby("A").transform(lambda x: (x - x.mean()) / x.std())
+        result = df.groupby("A", dropna=False).transform(
+            lambda x: (x - x.mean()) / x.std()
+        )
     expected = DataFrame(
         dict(B=np.nan, C=Series([-1, 0, 1, -1, 0, 1], dtype="float64"))
     )
@@ -612,8 +614,7 @@ def test_cython_transform_series(op, args, targop):
 
     # series
     for data in [s, s_missing]:
-        # print(data.head())
-        expected = data.groupby(labels).transform(targop)
+        expected = data.groupby(labels, dropna=False).transform(targop)
 
         tm.assert_series_equal(expected, data.groupby(labels).transform(op, *args))
         tm.assert_series_equal(expected, getattr(data.groupby(labels), op)(*args))