Skip to content

CLN: enforce deprecation of NDFrame.interpolate with ffill/bfill/pad/backfill methods #57869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

84 changes: 16 additions & 68 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -7630,7 +7630,6 @@ def interpolate(
* 'time': Works on daily and higher resolution data to interpolate
given length of interval.
* 'index', 'values': use the actual numerical values of the index.
* 'pad': Fill in NaNs using existing values.
* 'nearest', 'zero', 'slinear', 'quadratic', 'cubic',
'barycentric', 'polynomial': Passed to
`scipy.interpolate.interp1d`, whereas 'spline' is passed to
Expand All @@ -7654,23 +7653,9 @@ def interpolate(
0.
inplace : bool, default False
Update the data in place if possible.
limit_direction : {{'forward', 'backward', 'both'}}, Optional
limit_direction : {{'forward', 'backward', 'both'}}, optional, default 'forward'
Consecutive NaNs will be filled in this direction.

If limit is specified:
* If 'method' is 'pad' or 'ffill', 'limit_direction' must be 'forward'.
* If 'method' is 'backfill' or 'bfill', 'limit_direction' must be
'backwards'.

If 'limit' is not specified:
* If 'method' is 'backfill' or 'bfill', the default is 'backward'
* else the default is 'forward'

raises ValueError if `limit_direction` is 'forward' or 'both' and
method is 'backfill' or 'bfill'.
raises ValueError if `limit_direction` is 'backward' or 'both' and
method is 'pad' or 'ffill'.

limit_area : {{`None`, 'inside', 'outside'}}, default None
If limit is specified, consecutive NaNs will be filled with this
restriction.
Expand Down Expand Up @@ -7803,30 +7788,11 @@ def interpolate(
if not isinstance(method, str):
raise ValueError("'method' should be a string, not None.")

fillna_methods = ["ffill", "bfill", "pad", "backfill"]
if method.lower() in fillna_methods:
# GH#53581
warnings.warn(
f"{type(self).__name__}.interpolate with method={method} is "
"deprecated and will raise in a future version. "
"Use obj.ffill() or obj.bfill() instead.",
FutureWarning,
stacklevel=find_stack_level(),
)
obj, should_transpose = self, False
else:
obj, should_transpose = (self.T, True) if axis == 1 else (self, False)
# GH#53631
if np.any(obj.dtypes == object):
raise TypeError(
f"{type(self).__name__} cannot interpolate with object dtype."
)

if method in fillna_methods and "fill_value" in kwargs:
raise ValueError(
"'fill_value' is not a valid keyword for "
f"{type(self).__name__}.interpolate with method from "
f"{fillna_methods}"
obj, should_transpose = (self.T, True) if axis == 1 else (self, False)
# GH#53631
if np.any(obj.dtypes == object):
raise TypeError(
f"{type(self).__name__} cannot interpolate with object dtype."
)

if isinstance(obj.index, MultiIndex) and method != "linear":
Expand All @@ -7836,34 +7802,16 @@ def interpolate(

limit_direction = missing.infer_limit_direction(limit_direction, method)

if method.lower() in fillna_methods:
# TODO(3.0): remove this case
# TODO: warn/raise on limit_direction or kwargs which are ignored?
# as of 2023-06-26 no tests get here with either
if not self._mgr.is_single_block and axis == 1:
# GH#53898
if inplace:
raise NotImplementedError()
obj, axis, should_transpose = self.T, 1 - axis, True

new_data = obj._mgr.pad_or_backfill(
method=method,
axis=self._get_block_manager_axis(axis),
limit=limit,
limit_area=limit_area,
inplace=inplace,
)
else:
index = missing.get_interp_index(method, obj.index)
new_data = obj._mgr.interpolate(
method=method,
index=index,
limit=limit,
limit_direction=limit_direction,
limit_area=limit_area,
inplace=inplace,
**kwargs,
)
index = missing.get_interp_index(method, obj.index)
new_data = obj._mgr.interpolate(
method=method,
index=index,
limit=limit,
limit_direction=limit_direction,
limit_area=limit_area,
inplace=inplace,
**kwargs,
)

result = self._constructor_from_mgr(new_data, axes=new_data.axes)
if should_transpose:
Expand Down
3 changes: 3 additions & 0 deletions pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1383,6 +1383,9 @@ def interpolate(
limit_area: Literal["inside", "outside"] | None = None,
**kwargs,
) -> list[Block]:
valid = missing.NP_METHODS + missing.SP_METHODS
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A better place for this check (and the one you added in get_interp_index) in is probably in _interpolate_scipy_wrapper.

We want to make these checks at the lowest level possible

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks, I moved this check from pandas/core/internals/blocks.py to _interpolate_scipy_wrapper in pandas/core/missing.py.

I left the check in get_interp_index, otherwise, several tests fail.

if method not in valid:
raise ValueError(f"Can not interpolate with method={method}.")
inplace = validate_bool_kwarg(inplace, "inplace")
# error: Non-overlapping equality check [...]
if method == "asfreq": # type: ignore[comparison-overlap]
Expand Down
3 changes: 3 additions & 0 deletions pandas/core/missing.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,9 @@ def get_interp_index(method, index: Index) -> Index:
or isinstance(index.dtype, DatetimeTZDtype)
or lib.is_np_dtype(index.dtype, "mM")
)
valid = NP_METHODS + SP_METHODS
if method not in valid:
raise ValueError(f"Can not interpolate with method={method}.")
if method not in methods and not is_numeric_or_datetime:
raise ValueError(
"Index column must be numeric or datetime type when "
Expand Down
33 changes: 15 additions & 18 deletions pandas/tests/copy_view/test_interp_fillna.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,18 @@ def test_interpolate_no_op(method):
df = DataFrame({"a": [1, 2]})
df_orig = df.copy()

warn = None
if method == "pad":
warn = FutureWarning
msg = "DataFrame.interpolate with method=pad is deprecated"
with tm.assert_produces_warning(warn, match=msg):
msg = f"Can not interpolate with method={method}"
with pytest.raises(ValueError, match=msg):
df.interpolate(method=method)
else:
result = df.interpolate(method=method)
assert np.shares_memory(get_array(result, "a"), get_array(df, "a"))

assert np.shares_memory(get_array(result, "a"), get_array(df, "a"))
result.iloc[0, 0] = 100

result.iloc[0, 0] = 100

assert not np.shares_memory(get_array(result, "a"), get_array(df, "a"))
tm.assert_frame_equal(df, df_orig)
assert not np.shares_memory(get_array(result, "a"), get_array(df, "a"))
tm.assert_frame_equal(df, df_orig)


@pytest.mark.parametrize("func", ["ffill", "bfill"])
Expand Down Expand Up @@ -122,9 +121,6 @@ def test_interpolate_cannot_with_object_dtype():
def test_interpolate_object_convert_no_op():
df = DataFrame({"a": ["a", "b", "c"], "b": 1})
arr_a = get_array(df, "a")
msg = "DataFrame.interpolate with method=pad is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
df.interpolate(method="pad", inplace=True)

# Now CoW makes a copy, it should not!
assert df._mgr._has_no_reference(0)
Expand All @@ -134,8 +130,8 @@ def test_interpolate_object_convert_no_op():
def test_interpolate_object_convert_copies():
df = DataFrame({"a": [1, np.nan, 2.5], "b": 1})
arr_a = get_array(df, "a")
msg = "DataFrame.interpolate with method=pad is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
msg = "Can not interpolate with method=pad"
with pytest.raises(ValueError, match=msg):
df.interpolate(method="pad", inplace=True, downcast="infer")

assert df._mgr._has_no_reference(0)
Expand All @@ -147,12 +143,13 @@ def test_interpolate_downcast_reference_triggers_copy():
df_orig = df.copy()
arr_a = get_array(df, "a")
view = df[:]
msg = "DataFrame.interpolate with method=pad is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):

msg = "Can not interpolate with method=pad"
with pytest.raises(ValueError, match=msg):
df.interpolate(method="pad", inplace=True, downcast="infer")
assert df._mgr._has_no_reference(0)
assert not np.shares_memory(arr_a, get_array(df, "a"))

assert df._mgr._has_no_reference(0)
assert not np.shares_memory(arr_a, get_array(df, "a"))
tm.assert_frame_equal(df_orig, view)


Expand Down
17 changes: 4 additions & 13 deletions pandas/tests/frame/methods/test_interpolate.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,13 +129,7 @@ def test_interp_bad_method(self):
"C": [1, 2, 3, 5],
}
)
msg = (
r"method must be one of \['linear', 'time', 'index', 'values', "
r"'nearest', 'zero', 'slinear', 'quadratic', 'cubic', "
r"'barycentric', 'krogh', 'spline', 'polynomial', "
r"'from_derivatives', 'piecewise_polynomial', 'pchip', 'akima', "
r"'cubicspline'\]. Got 'not_a_method' instead."
)
msg = "Can not interpolate with method=not_a_method"
with pytest.raises(ValueError, match=msg):
df.interpolate(method="not_a_method")

Expand Down Expand Up @@ -398,12 +392,9 @@ def test_interp_fillna_methods(self, axis, multiblock, method):
df["D"] = np.nan
df["E"] = 1.0

method2 = method if method != "pad" else "ffill"
expected = getattr(df, method2)(axis=axis)
msg = f"DataFrame.interpolate with method={method} is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = df.interpolate(method=method, axis=axis)
tm.assert_frame_equal(result, expected)
msg = f"Can not interpolate with method={method}"
with pytest.raises(ValueError, match=msg):
df.interpolate(method=method, axis=axis)

def test_interpolate_empty_df(self):
# GH#53199
Expand Down
Loading