diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index ef561d50066d1..f60aa6e4f26b1 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -211,6 +211,7 @@ Removal of prior version deprecations/changes - Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Second`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`57627`) - Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting units in :class:`Timedelta` (:issue:`57627`) - Enforced deprecation of the behavior of :func:`concat` when ``len(keys) != len(objs)`` would truncate to the shorter of the two. Now this raises a ``ValueError`` (:issue:`43485`) +- Enforced deprecation of values "pad", "ffill", "bfill", and "backfill" for :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` (:issue:`57869`) - Enforced silent-downcasting deprecation for :ref:`all relevant methods ` (:issue:`54710`) - In :meth:`DataFrame.stack`, the default value of ``future_stack`` is now ``True``; specifying ``False`` will raise a ``FutureWarning`` (:issue:`55448`) - Iterating over a :class:`.DataFrameGroupBy` or :class:`.SeriesGroupBy` will return tuples of length 1 for the groups when grouping by ``level`` a list of length 1 (:issue:`50064`) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index c9e6ffe1d7dc6..c0eda7f022d8f 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -7624,7 +7624,6 @@ def interpolate( * 'time': Works on daily and higher resolution data to interpolate given length of interval. * 'index', 'values': use the actual numerical values of the index. - * 'pad': Fill in NaNs using existing values. * 'nearest', 'zero', 'slinear', 'quadratic', 'cubic', 'barycentric', 'polynomial': Passed to `scipy.interpolate.interp1d`, whereas 'spline' is passed to @@ -7648,23 +7647,9 @@ def interpolate( 0. inplace : bool, default False Update the data in place if possible. - limit_direction : {{'forward', 'backward', 'both'}}, Optional + limit_direction : {{'forward', 'backward', 'both'}}, optional, default 'forward' Consecutive NaNs will be filled in this direction. - If limit is specified: - * If 'method' is 'pad' or 'ffill', 'limit_direction' must be 'forward'. - * If 'method' is 'backfill' or 'bfill', 'limit_direction' must be - 'backwards'. - - If 'limit' is not specified: - * If 'method' is 'backfill' or 'bfill', the default is 'backward' - * else the default is 'forward' - - raises ValueError if `limit_direction` is 'forward' or 'both' and - method is 'backfill' or 'bfill'. - raises ValueError if `limit_direction` is 'backward' or 'both' and - method is 'pad' or 'ffill'. - limit_area : {{`None`, 'inside', 'outside'}}, default None If limit is specified, consecutive NaNs will be filled with this restriction. @@ -7797,30 +7782,11 @@ def interpolate( if not isinstance(method, str): raise ValueError("'method' should be a string, not None.") - fillna_methods = ["ffill", "bfill", "pad", "backfill"] - if method.lower() in fillna_methods: - # GH#53581 - warnings.warn( - f"{type(self).__name__}.interpolate with method={method} is " - "deprecated and will raise in a future version. " - "Use obj.ffill() or obj.bfill() instead.", - FutureWarning, - stacklevel=find_stack_level(), - ) - obj, should_transpose = self, False - else: - obj, should_transpose = (self.T, True) if axis == 1 else (self, False) - # GH#53631 - if np.any(obj.dtypes == object): - raise TypeError( - f"{type(self).__name__} cannot interpolate with object dtype." - ) - - if method in fillna_methods and "fill_value" in kwargs: - raise ValueError( - "'fill_value' is not a valid keyword for " - f"{type(self).__name__}.interpolate with method from " - f"{fillna_methods}" + obj, should_transpose = (self.T, True) if axis == 1 else (self, False) + # GH#53631 + if np.any(obj.dtypes == object): + raise TypeError( + f"{type(self).__name__} cannot interpolate with object dtype." ) if isinstance(obj.index, MultiIndex) and method != "linear": @@ -7830,34 +7796,16 @@ def interpolate( limit_direction = missing.infer_limit_direction(limit_direction, method) - if method.lower() in fillna_methods: - # TODO(3.0): remove this case - # TODO: warn/raise on limit_direction or kwargs which are ignored? - # as of 2023-06-26 no tests get here with either - if not self._mgr.is_single_block and axis == 1: - # GH#53898 - if inplace: - raise NotImplementedError() - obj, axis, should_transpose = self.T, 1 - axis, True - - new_data = obj._mgr.pad_or_backfill( - method=method, - axis=self._get_block_manager_axis(axis), - limit=limit, - limit_area=limit_area, - inplace=inplace, - ) - else: - index = missing.get_interp_index(method, obj.index) - new_data = obj._mgr.interpolate( - method=method, - index=index, - limit=limit, - limit_direction=limit_direction, - limit_area=limit_area, - inplace=inplace, - **kwargs, - ) + index = missing.get_interp_index(method, obj.index) + new_data = obj._mgr.interpolate( + method=method, + index=index, + limit=limit, + limit_direction=limit_direction, + limit_area=limit_area, + inplace=inplace, + **kwargs, + ) result = self._constructor_from_mgr(new_data, axes=new_data.axes) if should_transpose: diff --git a/pandas/core/missing.py b/pandas/core/missing.py index de26ad14a7b7a..b3e152e36a304 100644 --- a/pandas/core/missing.py +++ b/pandas/core/missing.py @@ -322,13 +322,17 @@ def get_interp_index(method, index: Index) -> Index: or isinstance(index.dtype, DatetimeTZDtype) or lib.is_np_dtype(index.dtype, "mM") ) - if method not in methods and not is_numeric_or_datetime: - raise ValueError( - "Index column must be numeric or datetime type when " - f"using {method} method other than linear. " - "Try setting a numeric or datetime index column before " - "interpolating." - ) + valid = NP_METHODS + SP_METHODS + if method in valid: + if method not in methods and not is_numeric_or_datetime: + raise ValueError( + "Index column must be numeric or datetime type when " + f"using {method} method other than linear. " + "Try setting a numeric or datetime index column before " + "interpolating." + ) + else: + raise ValueError(f"Can not interpolate with method={method}.") if isna(index).any(): raise NotImplementedError( @@ -611,7 +615,9 @@ def _interpolate_scipy_wrapper( y = y.copy() if not new_x.flags.writeable: new_x = new_x.copy() - terp = alt_methods[method] + terp = alt_methods.get(method, None) + if terp is None: + raise ValueError(f"Can not interpolate with method={method}.") new_y = terp(x, y, new_x, **kwargs) return new_y diff --git a/pandas/tests/copy_view/test_interp_fillna.py b/pandas/tests/copy_view/test_interp_fillna.py index 8fe58e59b9cfd..abd87162ec32e 100644 --- a/pandas/tests/copy_view/test_interp_fillna.py +++ b/pandas/tests/copy_view/test_interp_fillna.py @@ -19,19 +19,18 @@ def test_interpolate_no_op(method): df = DataFrame({"a": [1, 2]}) df_orig = df.copy() - warn = None if method == "pad": - warn = FutureWarning - msg = "DataFrame.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(warn, match=msg): + msg = f"Can not interpolate with method={method}" + with pytest.raises(ValueError, match=msg): + df.interpolate(method=method) + else: result = df.interpolate(method=method) + assert np.shares_memory(get_array(result, "a"), get_array(df, "a")) - assert np.shares_memory(get_array(result, "a"), get_array(df, "a")) + result.iloc[0, 0] = 100 - result.iloc[0, 0] = 100 - - assert not np.shares_memory(get_array(result, "a"), get_array(df, "a")) - tm.assert_frame_equal(df, df_orig) + assert not np.shares_memory(get_array(result, "a"), get_array(df, "a")) + tm.assert_frame_equal(df, df_orig) @pytest.mark.parametrize("func", ["ffill", "bfill"]) @@ -122,9 +121,6 @@ def test_interpolate_cannot_with_object_dtype(): def test_interpolate_object_convert_no_op(): df = DataFrame({"a": ["a", "b", "c"], "b": 1}) arr_a = get_array(df, "a") - msg = "DataFrame.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - df.interpolate(method="pad", inplace=True) # Now CoW makes a copy, it should not! assert df._mgr._has_no_reference(0) @@ -134,8 +130,8 @@ def test_interpolate_object_convert_no_op(): def test_interpolate_object_convert_copies(): df = DataFrame({"a": [1, np.nan, 2.5], "b": 1}) arr_a = get_array(df, "a") - msg = "DataFrame.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): df.interpolate(method="pad", inplace=True, downcast="infer") assert df._mgr._has_no_reference(0) @@ -147,12 +143,13 @@ def test_interpolate_downcast_reference_triggers_copy(): df_orig = df.copy() arr_a = get_array(df, "a") view = df[:] - msg = "DataFrame.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): + + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): df.interpolate(method="pad", inplace=True, downcast="infer") + assert df._mgr._has_no_reference(0) + assert not np.shares_memory(arr_a, get_array(df, "a")) - assert df._mgr._has_no_reference(0) - assert not np.shares_memory(arr_a, get_array(df, "a")) tm.assert_frame_equal(df_orig, view) diff --git a/pandas/tests/frame/methods/test_interpolate.py b/pandas/tests/frame/methods/test_interpolate.py index 2ba3bbd3109a2..0a9d059736e6f 100644 --- a/pandas/tests/frame/methods/test_interpolate.py +++ b/pandas/tests/frame/methods/test_interpolate.py @@ -129,13 +129,7 @@ def test_interp_bad_method(self): "C": [1, 2, 3, 5], } ) - msg = ( - r"method must be one of \['linear', 'time', 'index', 'values', " - r"'nearest', 'zero', 'slinear', 'quadratic', 'cubic', " - r"'barycentric', 'krogh', 'spline', 'polynomial', " - r"'from_derivatives', 'piecewise_polynomial', 'pchip', 'akima', " - r"'cubicspline'\]. Got 'not_a_method' instead." - ) + msg = "Can not interpolate with method=not_a_method" with pytest.raises(ValueError, match=msg): df.interpolate(method="not_a_method") @@ -398,12 +392,9 @@ def test_interp_fillna_methods(self, axis, multiblock, method): df["D"] = np.nan df["E"] = 1.0 - method2 = method if method != "pad" else "ffill" - expected = getattr(df, method2)(axis=axis) - msg = f"DataFrame.interpolate with method={method} is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = df.interpolate(method=method, axis=axis) - tm.assert_frame_equal(result, expected) + msg = f"Can not interpolate with method={method}" + with pytest.raises(ValueError, match=msg): + df.interpolate(method=method, axis=axis) def test_interpolate_empty_df(self): # GH#53199 diff --git a/pandas/tests/series/methods/test_interpolate.py b/pandas/tests/series/methods/test_interpolate.py index e4726f3ec6b32..c5df1fd498938 100644 --- a/pandas/tests/series/methods/test_interpolate.py +++ b/pandas/tests/series/methods/test_interpolate.py @@ -344,7 +344,7 @@ def test_interpolate_invalid_float_limit(self, nontemporal_method): def test_interp_invalid_method(self, invalid_method): s = Series([1, 3, np.nan, 12, np.nan, 25]) - msg = f"method must be one of.* Got '{invalid_method}' instead" + msg = "Can not interpolate with method=nonexistent_method" if invalid_method is None: msg = "'method' should be a string, not None" with pytest.raises(ValueError, match=msg): @@ -355,16 +355,6 @@ def test_interp_invalid_method(self, invalid_method): with pytest.raises(ValueError, match=msg): s.interpolate(method=invalid_method, limit=-1) - def test_interp_invalid_method_and_value(self): - # GH#36624 - ser = Series([1, 3, np.nan, 12, np.nan, 25]) - - msg = "'fill_value' is not a valid keyword for Series.interpolate" - msg2 = "Series.interpolate with method=pad" - with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning(FutureWarning, match=msg2): - ser.interpolate(fill_value=3, method="pad") - def test_interp_limit_forward(self): s = Series([1, 3, np.nan, np.nan, np.nan, 11]) @@ -455,107 +445,70 @@ def test_interp_limit_area(self): s.interpolate(method="linear", limit_area="abc") @pytest.mark.parametrize( - "method, limit_direction, expected", - [ - ("pad", "backward", "forward"), - ("ffill", "backward", "forward"), - ("backfill", "forward", "backward"), - ("bfill", "forward", "backward"), - ("pad", "both", "forward"), - ("ffill", "both", "forward"), - ("backfill", "both", "backward"), - ("bfill", "both", "backward"), - ], - ) - def test_interp_limit_direction_raises(self, method, limit_direction, expected): - # https://github.com/pandas-dev/pandas/pull/34746 - s = Series([1, 2, 3]) - - msg = f"`limit_direction` must be '{expected}' for method `{method}`" - msg2 = "Series.interpolate with method=" - with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning(FutureWarning, match=msg2): - s.interpolate(method=method, limit_direction=limit_direction) - - @pytest.mark.parametrize( - "data, expected_data, kwargs", + "data, kwargs", ( ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, 3.0, 3.0, 3.0, 7.0, np.nan, np.nan], {"method": "pad", "limit_area": "inside"}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, 3.0, np.nan, np.nan, 7.0, np.nan, np.nan], {"method": "pad", "limit_area": "inside", "limit": 1}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 7.0, 7.0, 7.0], {"method": "pad", "limit_area": "outside"}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 7.0, 7.0, np.nan], {"method": "pad", "limit_area": "outside", "limit": 1}, ), ( - [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], {"method": "pad", "limit_area": "outside", "limit": 1}, ), ( - range(5), range(5), {"method": "pad", "limit_area": "outside", "limit": 1}, ), ), ) - def test_interp_limit_area_with_pad(self, data, expected_data, kwargs): + def test_interp_limit_area_with_pad(self, data, kwargs): # GH26796 s = Series(data) - expected = Series(expected_data) - msg = "Series.interpolate with method=pad" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = s.interpolate(**kwargs) - tm.assert_series_equal(result, expected) + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): + s.interpolate(**kwargs) @pytest.mark.parametrize( - "data, expected_data, kwargs", + "data, kwargs", ( ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, 7.0, 7.0, 7.0, 7.0, np.nan, np.nan], {"method": "bfill", "limit_area": "inside"}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, np.nan, 3.0, np.nan, np.nan, 7.0, 7.0, np.nan, np.nan], {"method": "bfill", "limit_area": "inside", "limit": 1}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [3.0, 3.0, 3.0, np.nan, np.nan, np.nan, 7.0, np.nan, np.nan], {"method": "bfill", "limit_area": "outside"}, ), ( [np.nan, np.nan, 3, np.nan, np.nan, np.nan, 7, np.nan, np.nan], - [np.nan, 3.0, 3.0, np.nan, np.nan, np.nan, 7.0, np.nan, np.nan], {"method": "bfill", "limit_area": "outside", "limit": 1}, ), ), ) - def test_interp_limit_area_with_backfill(self, data, expected_data, kwargs): + def test_interp_limit_area_with_backfill(self, data, kwargs): # GH26796 - s = Series(data) - expected = Series(expected_data) - msg = "Series.interpolate with method=bfill" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = s.interpolate(**kwargs) - tm.assert_series_equal(result, expected) + + msg = "Can not interpolate with method=bfill" + with pytest.raises(ValueError, match=msg): + s.interpolate(**kwargs) def test_interp_limit_direction(self): # These tests are for issue #9218 -- fill NaNs in both directions. @@ -650,20 +603,18 @@ def test_interp_datetime64(self, method, tz_naive_fixture): df = Series( [1, np.nan, 3], index=date_range("1/1/2000", periods=3, tz=tz_naive_fixture) ) - warn = None if method == "nearest" else FutureWarning - msg = "Series.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(warn, match=msg): - result = df.interpolate(method=method) - if warn is not None: - # check the "use ffill instead" is equivalent - alt = df.ffill() - tm.assert_series_equal(result, alt) - expected = Series( - [1.0, 1.0, 3.0], - index=date_range("1/1/2000", periods=3, tz=tz_naive_fixture), - ) - tm.assert_series_equal(result, expected) + if method == "nearest": + result = df.interpolate(method=method) + expected = Series( + [1.0, 1.0, 3.0], + index=date_range("1/1/2000", periods=3, tz=tz_naive_fixture), + ) + tm.assert_series_equal(result, expected) + else: + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): + df.interpolate(method=method) def test_interp_pad_datetime64tz_values(self): # GH#27628 missing.interpolate_2d should handle datetimetz values @@ -671,16 +622,9 @@ def test_interp_pad_datetime64tz_values(self): ser = Series(dti) ser[1] = pd.NaT - msg = "Series.interpolate with method=pad is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = ser.interpolate(method="pad") - # check the "use ffill instead" is equivalent - alt = ser.ffill() - tm.assert_series_equal(result, alt) - - expected = Series(dti) - expected[1] = expected[0] - tm.assert_series_equal(result, expected) + msg = "Can not interpolate with method=pad" + with pytest.raises(ValueError, match=msg): + ser.interpolate(method="pad") def test_interp_limit_no_nans(self): # GH 7173