Skip to content

Commit b5d666c

Browse files
authored
BUG: interpolate with fillna methods fail to fill across multiblocks (#53962)
* BUG: interpolate with fillna methods fail to fill across multiblocks * changelog added * resolved conversation * reorganize code; resolve conversations * retrigger checks * try to resolve conversation * fix failing tests * resolve conversation
1 parent c126eeb commit b5d666c

File tree

3 files changed

+39
-27
lines changed

3 files changed

+39
-27
lines changed

doc/source/whatsnew/v2.1.0.rst

+1
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,7 @@ Indexing
446446

447447
Missing
448448
^^^^^^^
449+
- Bug in :meth:`DataFrame.interpolate` failing to fill across multiblock data when ``method`` is "pad", "ffill", "bfill", or "backfill" (:issue:`53898`)
449450
- Bug in :meth:`DataFrame.interpolate` ignoring ``inplace`` when :class:`DataFrame` is empty (:issue:`53199`)
450451
- Bug in :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` failing to raise on invalid ``downcast`` keyword, which can be only ``None`` or "infer" (:issue:`53103`)
451452
- Bug in :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` with complex dtype incorrectly failing to fill ``NaN`` entries (:issue:`53635`)

pandas/core/generic.py

+30-26
Original file line numberDiff line numberDiff line change
@@ -7980,22 +7980,18 @@ def interpolate(
79807980
raise ValueError("downcast must be either None or 'infer'")
79817981

79827982
inplace = validate_bool_kwarg(inplace, "inplace")
7983-
79847983
axis = self._get_axis_number(axis)
79857984

7986-
fillna_methods = ["ffill", "bfill", "pad", "backfill"]
7987-
should_transpose = axis == 1 and method not in fillna_methods
7988-
7989-
obj = self.T if should_transpose else self
7990-
7991-
if obj.empty:
7985+
if self.empty:
79927986
if inplace:
79937987
return None
79947988
return self.copy()
79957989

79967990
if not isinstance(method, str):
79977991
raise ValueError("'method' should be a string, not None.")
7998-
elif method.lower() in fillna_methods:
7992+
7993+
fillna_methods = ["ffill", "bfill", "pad", "backfill"]
7994+
if method.lower() in fillna_methods:
79997995
# GH#53581
80007996
warnings.warn(
80017997
f"{type(self).__name__}.interpolate with method={method} is "
@@ -8004,17 +8000,26 @@ def interpolate(
80048000
FutureWarning,
80058001
stacklevel=find_stack_level(),
80068002
)
8007-
elif np.any(obj.dtypes == object):
8008-
# GH#53631
8009-
if not (obj.ndim == 2 and np.all(obj.dtypes == object)):
8010-
# don't warn in cases that already raise
8011-
warnings.warn(
8012-
f"{type(self).__name__}.interpolate with object dtype is "
8013-
"deprecated and will raise in a future version. Call "
8014-
"obj.infer_objects(copy=False) before interpolating instead.",
8015-
FutureWarning,
8016-
stacklevel=find_stack_level(),
8017-
)
8003+
obj, should_transpose = self, False
8004+
else:
8005+
obj, should_transpose = (self.T, True) if axis == 1 else (self, False)
8006+
if np.any(obj.dtypes == object):
8007+
# GH#53631
8008+
if not (obj.ndim == 2 and np.all(obj.dtypes == object)):
8009+
# don't warn in cases that already raise
8010+
warnings.warn(
8011+
f"{type(self).__name__}.interpolate with object dtype is "
8012+
"deprecated and will raise in a future version. Call "
8013+
"obj.infer_objects(copy=False) before interpolating instead.",
8014+
FutureWarning,
8015+
stacklevel=find_stack_level(),
8016+
)
8017+
8018+
if "fill_value" in kwargs:
8019+
raise ValueError(
8020+
"'fill_value' is not a valid keyword for "
8021+
f"{type(self).__name__}.interpolate"
8022+
)
80188023

80198024
if isinstance(obj.index, MultiIndex) and method != "linear":
80208025
raise ValueError(
@@ -8023,23 +8028,22 @@ def interpolate(
80238028

80248029
limit_direction = missing.infer_limit_direction(limit_direction, method)
80258030

8026-
if obj.ndim == 2 and np.all(obj.dtypes == np.dtype("object")):
8031+
if obj.ndim == 2 and np.all(obj.dtypes == object):
80278032
raise TypeError(
80288033
"Cannot interpolate with all object-dtype columns "
80298034
"in the DataFrame. Try setting at least one "
80308035
"column to a numeric dtype."
80318036
)
80328037

8033-
if "fill_value" in kwargs:
8034-
raise ValueError(
8035-
"'fill_value' is not a valid keyword for "
8036-
f"{type(self).__name__}.interpolate"
8037-
)
8038-
80398038
if method.lower() in fillna_methods:
80408039
# TODO(3.0): remove this case
80418040
# TODO: warn/raise on limit_direction or kwargs which are ignored?
80428041
# as of 2023-06-26 no tests get here with either
8042+
if not self._mgr.is_single_block and axis == 1:
8043+
# GH#53898
8044+
if inplace:
8045+
raise NotImplementedError()
8046+
obj, axis, should_transpose = self.T, 1 - axis, True
80438047

80448048
new_data = obj._mgr.pad_or_backfill(
80458049
method=method,

pandas/tests/frame/methods/test_interpolate.py

+8-1
Original file line numberDiff line numberDiff line change
@@ -451,8 +451,11 @@ def test_interp_string_axis(self, axis_name, axis_number):
451451
expected = df.interpolate(method="linear", axis=axis_number)
452452
tm.assert_frame_equal(result, expected)
453453

454+
@pytest.mark.parametrize("multiblock", [True, False])
454455
@pytest.mark.parametrize("method", ["ffill", "bfill", "pad"])
455-
def test_interp_fillna_methods(self, request, axis, method, using_array_manager):
456+
def test_interp_fillna_methods(
457+
self, request, axis, multiblock, method, using_array_manager
458+
):
456459
# GH 12918
457460
if using_array_manager and axis in (1, "columns"):
458461
# TODO(ArrayManager) support axis=1
@@ -465,6 +468,10 @@ def test_interp_fillna_methods(self, request, axis, method, using_array_manager)
465468
"C": [3.0, 6.0, 9.0, np.nan, np.nan, 30.0],
466469
}
467470
)
471+
if multiblock:
472+
df["D"] = np.nan
473+
df["E"] = 1.0
474+
468475
method2 = method if method != "pad" else "ffill"
469476
expected = getattr(df, method2)(axis=axis)
470477
msg = f"DataFrame.interpolate with method={method} is deprecated"

0 commit comments

Comments
 (0)