From 241247e2cb6ec5a4a315d4d5fb282fbddcfa592c Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 12 Jun 2023 08:53:17 -0700 Subject: [PATCH 1/4] BUG: clean_fill_method failing to raise --- doc/source/whatsnew/v2.1.0.rst | 1 + pandas/core/generic.py | 4 +++- pandas/core/internals/blocks.py | 3 +++ pandas/core/missing.py | 8 +++----- pandas/core/resample.py | 4 ++++ pandas/tests/copy_view/test_interp_fillna.py | 2 +- pandas/tests/frame/methods/test_align.py | 8 ++++++++ pandas/tests/frame/methods/test_reindex.py | 7 +++++++ pandas/tests/series/methods/test_interpolate.py | 6 ++++++ 9 files changed, 36 insertions(+), 7 deletions(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index baacc8c421414..2c7040ba75c2b 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -505,6 +505,7 @@ Other - Bug in :meth:`Series.map` when giving a callable to an empty series, the returned series had ``object`` dtype. It now keeps the original dtype (:issue:`52384`) - Bug in :meth:`Series.memory_usage` when ``deep=True`` throw an error with Series of objects and the returned value is incorrect, as it does not take into account GC corrections (:issue:`51858`) - Fixed incorrect ``__name__`` attribute of ``pandas._libs.json`` (:issue:`52898`) +- Bug in :meth:`Series.align`, :meth:`DataFrame.align`, :meth:`Series.reindex`, :meth:`DataFrame.reindex`,:meth:`Series.interpolate`, :meth:`DataFrame.interpolate`, incorrectly failing to raise with method="asfreq" (:issue:`??`) .. ***DO NOT USE THIS SECTION*** diff --git a/pandas/core/generic.py b/pandas/core/generic.py index f73ef36f76086..0c0f1a332c691 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -9735,7 +9735,9 @@ def align( method = None if limit is lib.no_default: limit = None - method = clean_fill_method(method) + + if method is not None: + method = clean_fill_method(method) if broadcast_axis is not lib.no_default: # GH#51856 diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 7c5d686d96939..335a7861260fc 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1369,6 +1369,9 @@ def interpolate( m = missing.clean_fill_method(method) except ValueError: m = None + if method == "asfreq": + # clean_fill_method used to allow this + raise if m is None and self.dtype.kind != "f": # only deal with floats # bc we already checked that can_hold_na, we don't have int dtype here diff --git a/pandas/core/missing.py b/pandas/core/missing.py index 7762ba8e2c730..f38a38eee9a62 100644 --- a/pandas/core/missing.py +++ b/pandas/core/missing.py @@ -122,11 +122,7 @@ def mask_missing(arr: ArrayLike, values_to_mask) -> npt.NDArray[np.bool_]: return mask -def clean_fill_method(method: str | None, allow_nearest: bool = False): - # asfreq is compat for resampling - if method in [None, "asfreq"]: - return None - +def clean_fill_method(method: str, allow_nearest: bool = False): if isinstance(method, str): method = method.lower() if method == "ffill": @@ -954,6 +950,8 @@ def get_fill_func(method, ndim: int = 1): def clean_reindex_fill_method(method) -> ReindexMethod | None: + if method is None: + return None return clean_fill_method(method, allow_nearest=True) diff --git a/pandas/core/resample.py b/pandas/core/resample.py index 8291162db0834..a9faad4cbef6a 100644 --- a/pandas/core/resample.py +++ b/pandas/core/resample.py @@ -1506,6 +1506,8 @@ def _upsample(self, method, limit: int | None = None, fill_value=None): result = obj.copy() result.index = res_index else: + if method == "asfreq": + method = None result = obj.reindex( res_index, method=method, limit=limit, fill_value=fill_value ) @@ -1624,6 +1626,8 @@ def _upsample(self, method, limit: int | None = None, fill_value=None): memb = ax.asfreq(self.freq, how=self.convention) # Get the fill indexer + if method == "asfreq": + method = None indexer = memb.get_indexer(new_index, method=method, limit=limit) new_obj = _take_new_index( obj, diff --git a/pandas/tests/copy_view/test_interp_fillna.py b/pandas/tests/copy_view/test_interp_fillna.py index 576d3a9cdedde..2db9b8cd0ea0d 100644 --- a/pandas/tests/copy_view/test_interp_fillna.py +++ b/pandas/tests/copy_view/test_interp_fillna.py @@ -108,7 +108,7 @@ def test_interpolate_cleaned_fill_method(using_copy_on_write): df = DataFrame({"a": ["a", np.nan, "c"], "b": 1}) df_orig = df.copy() - result = df.interpolate(method="asfreq") + result = df.interpolate(method="linear") if using_copy_on_write: assert np.shares_memory(get_array(result, "a"), get_array(df, "a")) diff --git a/pandas/tests/frame/methods/test_align.py b/pandas/tests/frame/methods/test_align.py index 1dabc95a0f6f4..e56d542972e63 100644 --- a/pandas/tests/frame/methods/test_align.py +++ b/pandas/tests/frame/methods/test_align.py @@ -14,6 +14,14 @@ class TestDataFrameAlign: + def test_align_asfreq_method_raises(self): + df = DataFrame({"A": [1, np.nan, 2]}) + msg = "Invalid fill method" + msg2 = "The 'method', 'limit', and 'fill_axis' keywords" + with pytest.raises(ValueError, match=msg): + with tm.assert_produces_warning(FutureWarning, match=msg2): + df.align(df.iloc[::-1], method="asfreq") + def test_frame_align_aware(self): idx1 = date_range("2001", periods=5, freq="H", tz="US/Eastern") idx2 = date_range("2001", periods=5, freq="2H", tz="US/Eastern") diff --git a/pandas/tests/frame/methods/test_reindex.py b/pandas/tests/frame/methods/test_reindex.py index e8cebd5964236..63e2eb790a4ea 100644 --- a/pandas/tests/frame/methods/test_reindex.py +++ b/pandas/tests/frame/methods/test_reindex.py @@ -1291,3 +1291,10 @@ def test_reindex_not_category(self, index_df, index_res, index_exp): result = df.reindex(index=index_res) expected = DataFrame(index=index_exp) tm.assert_frame_equal(result, expected) + + def test_invalid_method(self): + df = DataFrame({"A": [1, np.nan, 2]}) + + msg = "Invalid fill method" + with pytest.raises(ValueError, match=msg): + df.reindex([1, 0, 2], method="asfreq") diff --git a/pandas/tests/series/methods/test_interpolate.py b/pandas/tests/series/methods/test_interpolate.py index 6f4c4ba4dd69d..1b0a28f3940cc 100644 --- a/pandas/tests/series/methods/test_interpolate.py +++ b/pandas/tests/series/methods/test_interpolate.py @@ -821,3 +821,9 @@ def test_interpolate_unsorted_index(self, ascending, expected_values): result = ts.sort_index(ascending=ascending).interpolate(method="index") expected = Series(data=expected_values, index=expected_values, dtype=float) tm.assert_series_equal(result, expected) + + def test_interpolate_afreq_raises(self): + ser = Series(["a", None, "b"], dtype=object) + msg = "Invalid fill method" + with pytest.raises(ValueError, match=msg): + ser.interpolate(method="asfreq") From 22aac568360937047d9a461ce314ca60c004bf08 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 12 Jun 2023 09:37:50 -0700 Subject: [PATCH 2/4] mypy fixup --- pandas/core/internals/blocks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 335a7861260fc..3139f5bc06a00 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1369,7 +1369,10 @@ def interpolate( m = missing.clean_fill_method(method) except ValueError: m = None - if method == "asfreq": + # error: Non-overlapping equality check (left operand type: + # "Literal['backfill', 'bfill', 'ffill', 'pad']", right + # operand type: "Literal['asfreq']") + if method == "asfreq": # type: ignore[comparison-overlap] # clean_fill_method used to allow this raise if m is None and self.dtype.kind != "f": From e07f9f46bcb75e71a5d10f2392eb7f6349ad8248 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 12 Jun 2023 11:27:45 -0700 Subject: [PATCH 3/4] Update doc/source/whatsnew/v2.1.0.rst Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> --- doc/source/whatsnew/v2.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 2c7040ba75c2b..1fb7855ee946d 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -505,7 +505,7 @@ Other - Bug in :meth:`Series.map` when giving a callable to an empty series, the returned series had ``object`` dtype. It now keeps the original dtype (:issue:`52384`) - Bug in :meth:`Series.memory_usage` when ``deep=True`` throw an error with Series of objects and the returned value is incorrect, as it does not take into account GC corrections (:issue:`51858`) - Fixed incorrect ``__name__`` attribute of ``pandas._libs.json`` (:issue:`52898`) -- Bug in :meth:`Series.align`, :meth:`DataFrame.align`, :meth:`Series.reindex`, :meth:`DataFrame.reindex`,:meth:`Series.interpolate`, :meth:`DataFrame.interpolate`, incorrectly failing to raise with method="asfreq" (:issue:`??`) +- Bug in :meth:`Series.align`, :meth:`DataFrame.align`, :meth:`Series.reindex`, :meth:`DataFrame.reindex`, :meth:`Series.interpolate`, :meth:`DataFrame.interpolate`, incorrectly failing to raise with method="asfreq" (:issue:`53620`) .. ***DO NOT USE THIS SECTION*** From aa47c3efb8af6efb75ffaa4a0d301e025fffe0f7 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 12 Jun 2023 11:46:26 -0700 Subject: [PATCH 4/4] sort note --- doc/source/whatsnew/v2.1.0.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 1fb7855ee946d..13ba53682144f 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -502,10 +502,11 @@ Other - Bug in :func:`assert_almost_equal` now throwing assertion error for two unequal sets (:issue:`51727`) - Bug in :func:`assert_frame_equal` checks category dtypes even when asked not to check index type (:issue:`52126`) - Bug in :meth:`DataFrame.reindex` with a ``fill_value`` that should be inferred with a :class:`ExtensionDtype` incorrectly inferring ``object`` dtype (:issue:`52586`) +- Bug in :meth:`Series.align`, :meth:`DataFrame.align`, :meth:`Series.reindex`, :meth:`DataFrame.reindex`, :meth:`Series.interpolate`, :meth:`DataFrame.interpolate`, incorrectly failing to raise with method="asfreq" (:issue:`53620`) - Bug in :meth:`Series.map` when giving a callable to an empty series, the returned series had ``object`` dtype. It now keeps the original dtype (:issue:`52384`) - Bug in :meth:`Series.memory_usage` when ``deep=True`` throw an error with Series of objects and the returned value is incorrect, as it does not take into account GC corrections (:issue:`51858`) - Fixed incorrect ``__name__`` attribute of ``pandas._libs.json`` (:issue:`52898`) -- Bug in :meth:`Series.align`, :meth:`DataFrame.align`, :meth:`Series.reindex`, :meth:`DataFrame.reindex`, :meth:`Series.interpolate`, :meth:`DataFrame.interpolate`, incorrectly failing to raise with method="asfreq" (:issue:`53620`) +- .. ***DO NOT USE THIS SECTION***