BUG: groupby ffill adds labels as extra column (#21521)

doc/source/whatsnew/v0.25.0.rst

@@ -153,6 +153,37 @@ Providing any ``SparseSeries`` or ``SparseDataFrame`` to :func:`concat` will
 cause a ``SparseSeries`` or ``SparseDataFrame`` to be returned, as before.
 
 
+``DataFrame`` groupby ffill/bfill no longer return group labels
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The methods ``ffill``, ``bfill``, ``pad`` and ``backfill`` of
+:class:`DataFrameGroupBy <pandas.core.groupby.DataFrameGroupBy>`
+previously included the group labels in the return value, which was
+inconsistent with other groupby transforms. Now only the filled values
+are returned. (:issue:`21521`)
+
+.. ipython:: python
+
+    df = pd.DataFrame({"a": ["x", "y"], "b": [1, 2]})
+    df
+
+*Previous Behaviour*:
+
+.. code-block:: python
+
+   In [3]: df.groupby("a").ffill()
+   Out[3]:
+      a  b
+   0  x  1
+   1  y  2
+
+*New Behaviour*:
+
+.. ipython:: python
+
+    df.groupby("a").ffill()
+
+
 .. _whatsnew_0250.api_breaking.deps:
 
 Increased minimum versions for dependencies

pandas/core/groupby/generic.py

@@ -1480,15 +1480,6 @@ def _apply_to_column_groupbys(self, func):
              in self._iterate_column_groupbys()),
             keys=self._selected_obj.columns, axis=1)
 
-    def _fill(self, direction, limit=None):
-        """Overridden method to join grouped columns in output"""
-        res = super()._fill(direction, limit=limit)
-        output = OrderedDict(
-            (grp.name, grp.grouper) for grp in self.grouper.groupings)
-
-        from pandas import concat
-        return concat((self._wrap_transformed_output(output), res), axis=1)
-
     def count(self):
         """
         Compute count of group, excluding missing values.

pandas/core/groupby/groupby.py

@@ -2235,7 +2235,6 @@ def pct_change(self, periods=1, fill_method='pad', limit=None, freq=None,
                                                      limit=limit, freq=freq,
                                                      axis=axis))
         filled = getattr(self, fill_method)(limit=limit)
-        filled = filled.drop(self.grouper.names, axis=1)
         fill_grp = filled.groupby(self.grouper.labels)
         shifted = fill_grp.shift(periods=periods, freq=freq)
         return (filled / shifted) - 1

pandas/tests/groupby/test_transform.py

@@ -749,7 +749,7 @@ def interweave(list_obj):
         assert_series_equal(result, exp)
     else:
         result = getattr(df.groupby('key'), fill_method)(limit=limit)
-        exp = DataFrame({'key': keys, 'val': _exp_vals})
+        exp = DataFrame({'val': _exp_vals})
         assert_frame_equal(result, exp)
 
 
@@ -763,7 +763,7 @@ def test_pad_stable_sorting(fill_method):
         y = y[::-1]
 
     df = pd.DataFrame({'x': x, 'y': y})
-    expected = df.copy()
+    expected = df.drop('x', 1)
 
     result = getattr(df.groupby('x'), fill_method)()
 
@@ -789,7 +789,7 @@ def test_pct_change(test_series, freq, periods, fill_method, limit):
     df = DataFrame({'key': key_v, 'vals': vals * 2})
 
     df_g = getattr(df.groupby('key'), fill_method)(limit=limit)
-    grp = df_g.groupby('key')
+    grp = df_g.groupby(df.key)
 
     expected = grp['vals'].obj / grp['vals'].shift(periods) - 1
 
@@ -880,3 +880,14 @@ def test_transform_absent_categories(func):
     result = getattr(df.y.groupby(df.x), func)()
     expected = df.y
     assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize('func', ['ffill', 'bfill', 'shift'])
+@pytest.mark.parametrize('key, val', [('level', 0), ('by', Series([0]))])
+def test_ffill_not_in_axis(func, key, val):
+    # GH 21521
+    df = pd.DataFrame([[np.nan]])
+    result = getattr(df.groupby(**{key: val}), func)()
+    expected = df
+
+    assert_frame_equal(result, expected)