From bd3bbfc517698d8e148429d3c44429f8b5b74c6a Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 23 Jan 2023 15:39:50 -0800 Subject: [PATCH 1/4] DEPR: dt64 any/all GH#34479 --- doc/source/whatsnew/v2.0.0.rst | 2 ++ pandas/core/arrays/datetimelike.py | 5 ++-- pandas/core/nanops.py | 19 ++++++++++++++ pandas/tests/frame/test_reductions.py | 19 +++++++++++--- pandas/tests/reductions/test_reductions.py | 29 +++++++++++++--------- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index 7f983c97691ca..0e756f6f08751 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -578,6 +578,8 @@ Deprecations - :meth:`Index.is_floating` has been deprecated. Use :func:`pandas.api.types.is_float_dtype` instead (:issue:`50042`) - :meth:`Index.holds_integer` has been deprecated. Use :func:`pandas.api.types.infer_dtype` instead (:issue:`50243`) - :meth:`Index.is_categorical` has been deprecated. Use :func:`pandas.api.types.is_categorical_dtype` instead (:issue:`50042`) +- Deprecated ``all`` and ``any`` reductions with ``datetime64`` and :class:`DatetimeTZDtype` dtypes, use e.g. ``(obj != pd.Timestamp(0), tz=obj.tz).all()`` instead (:issue:`??`) +- .. --------------------------------------------------------------------------- .. _whatsnew_200.prior_deprecations: diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index bb21bed2dc779..aded9dc64d63b 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -2029,11 +2029,12 @@ def ceil( # Reductions def any(self, *, axis: AxisInt | None = None, skipna: bool = True) -> bool: - # GH#34479 discussion of desired behavior long-term + # GH#34479 the nanops call will issue a FutureWarning for non-td64 dtype return nanops.nanany(self._ndarray, axis=axis, skipna=skipna, mask=self.isna()) def all(self, *, axis: AxisInt | None = None, skipna: bool = True) -> bool: - # GH#34479 discussion of desired behavior long-term + # GH#34479 the nanops call will issue a FutureWarning for non-td64 dtype + return nanops.nanall(self._ndarray, axis=axis, skipna=skipna, mask=self.isna()) # -------------------------------------------------------------- diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 02372356d3fe4..9979d12af1f75 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -32,6 +32,7 @@ npt, ) from pandas.compat._optional import import_optional_dependency +from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.common import ( is_any_int_dtype, @@ -529,6 +530,15 @@ def nanany( >>> nanops.nanany(s) False """ + if needs_i8_conversion(values.dtype) and values.dtype.kind != "m": + # GH#34479 + warnings.warn( + "'any' with datetime64 dtypes is deprecated and will raise in a " + "future version. Use (obj != pd.Timestamp(0)).any() instead.", + FutureWarning, + stacklevel=find_stack_level(), + ) + values, _, _, _, _ = _get_values(values, skipna, fill_value=False, mask=mask) # For object type, any won't necessarily return @@ -575,6 +585,15 @@ def nanall( >>> nanops.nanall(s) False """ + if needs_i8_conversion(values.dtype) and values.dtype.kind != "m": + # GH#34479 + warnings.warn( + "'all' with datetime64 dtypes is deprecated and will raise in a " + "future version. Use (obj != pd.Timestamp(0)).all() instead.", + FutureWarning, + stacklevel=find_stack_level(), + ) + values, _, _, _, _ = _get_values(values, skipna, fill_value=True, mask=mask) # For object type, all won't necessarily return diff --git a/pandas/tests/frame/test_reductions.py b/pandas/tests/frame/test_reductions.py index 2d395a7cbd608..6dd5887d82efa 100644 --- a/pandas/tests/frame/test_reductions.py +++ b/pandas/tests/frame/test_reductions.py @@ -1150,7 +1150,10 @@ def test_any_datetime(self): ] df = DataFrame({"A": float_data, "B": datetime_data}) - result = df.any(axis=1) + with tm.assert_produces_warning(FutureWarning, match="is deprecated"): + # GH#34479 + result = df.any(axis=1) + expected = Series([True, True, True, False]) tm.assert_series_equal(result, expected) @@ -1245,12 +1248,22 @@ def test_any_all_np_func(self, func, data, expected): ): getattr(DataFrame(data), func.__name__)(axis=None) else: - result = func(data) + msg = "'(any|all)' with datetime64 dtypes is deprecated" + if data.dtypes.apply(lambda x: x.kind == "M").any(): + warn = FutureWarning + else: + warn = None + + with tm.assert_produces_warning(warn, match=msg, check_stacklevel=False): + # GH#34479 + result = func(data) assert isinstance(result, np.bool_) assert result.item() is expected # method version - result = getattr(DataFrame(data), func.__name__)(axis=None) + with tm.assert_produces_warning(warn, match=msg): + # GH#34479 + result = getattr(DataFrame(data), func.__name__)(axis=None) assert isinstance(result, np.bool_) assert result.item() is expected diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py index d8b9082ec318a..6116f37977b8c 100644 --- a/pandas/tests/reductions/test_reductions.py +++ b/pandas/tests/reductions/test_reductions.py @@ -985,27 +985,32 @@ def test_any_all_datetimelike(self): ser = Series(dta) df = DataFrame(ser) - assert dta.all() - assert dta.any() + msg = "'(any|all)' with datetime64 dtypes is deprecated" + with tm.assert_produces_warning(FutureWarning, match=msg): + # GH#34479 + assert dta.all() + assert dta.any() - assert ser.all() - assert ser.any() + assert ser.all() + assert ser.any() - assert df.any().all() - assert df.all().all() + assert df.any().all() + assert df.all().all() dta = dta.tz_localize("UTC") ser = Series(dta) df = DataFrame(ser) - assert dta.all() - assert dta.any() + with tm.assert_produces_warning(FutureWarning, match=msg): + # GH#34479 + assert dta.all() + assert dta.any() - assert ser.all() - assert ser.any() + assert ser.all() + assert ser.any() - assert df.any().all() - assert df.all().all() + assert df.any().all() + assert df.all().all() tda = dta - dta[0] ser = Series(tda) From 21db7cee0725ae2673505122acc5c2f70dfd8331 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 23 Jan 2023 15:41:03 -0800 Subject: [PATCH 2/4] GH ref --- doc/source/whatsnew/v2.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index 0e756f6f08751..35a28b08151d4 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -578,7 +578,7 @@ Deprecations - :meth:`Index.is_floating` has been deprecated. Use :func:`pandas.api.types.is_float_dtype` instead (:issue:`50042`) - :meth:`Index.holds_integer` has been deprecated. Use :func:`pandas.api.types.infer_dtype` instead (:issue:`50243`) - :meth:`Index.is_categorical` has been deprecated. Use :func:`pandas.api.types.is_categorical_dtype` instead (:issue:`50042`) -- Deprecated ``all`` and ``any`` reductions with ``datetime64`` and :class:`DatetimeTZDtype` dtypes, use e.g. ``(obj != pd.Timestamp(0), tz=obj.tz).all()`` instead (:issue:`??`) +- Deprecated ``all`` and ``any`` reductions with ``datetime64`` and :class:`DatetimeTZDtype` dtypes, use e.g. ``(obj != pd.Timestamp(0), tz=obj.tz).all()`` instead (:issue:`34479`) - .. --------------------------------------------------------------------------- From 47461055a75bfbaaa2b2d7ee77e8282968b641b6 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 23 Jan 2023 18:43:53 -0800 Subject: [PATCH 3/4] troubleshoot --- pandas/tests/frame/test_reductions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/tests/frame/test_reductions.py b/pandas/tests/frame/test_reductions.py index 6dd5887d82efa..3c5ea16963308 100644 --- a/pandas/tests/frame/test_reductions.py +++ b/pandas/tests/frame/test_reductions.py @@ -1150,9 +1150,8 @@ def test_any_datetime(self): ] df = DataFrame({"A": float_data, "B": datetime_data}) - with tm.assert_produces_warning(FutureWarning, match="is deprecated"): - # GH#34479 - result = df.any(axis=1) + # GH#50947 deprecates this, but it is not emitting a warning in some builds + result = df.any(axis=1) expected = Series([True, True, True, False]) tm.assert_series_equal(result, expected) From 692c5965adecbb24f34cbedd3b8d80b2c156abd0 Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 25 Jan 2023 16:01:12 -0800 Subject: [PATCH 4/4] filter warning --- pandas/tests/frame/test_reductions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_reductions.py b/pandas/tests/frame/test_reductions.py index 3c5ea16963308..3e6074971352d 100644 --- a/pandas/tests/frame/test_reductions.py +++ b/pandas/tests/frame/test_reductions.py @@ -1138,6 +1138,10 @@ def test_any_all_object_dtype(self, axis, bool_agg_func, skipna): expected = Series([True, True, True, True]) tm.assert_series_equal(result, expected) + # GH#50947 deprecates this but it is not emitting a warning in some builds. + @pytest.mark.filterwarnings( + "ignore:'any' with datetime64 dtypes is deprecated.*:FutureWarning" + ) def test_any_datetime(self): # GH 23070 @@ -1150,7 +1154,6 @@ def test_any_datetime(self): ] df = DataFrame({"A": float_data, "B": datetime_data}) - # GH#50947 deprecates this, but it is not emitting a warning in some builds result = df.any(axis=1) expected = Series([True, True, True, False])