diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 7edc3c50de96d..aa18e1182dccc 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -112,6 +112,7 @@ Deprecations Removal of prior version deprecations/changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :func:`read_excel`, :func:`read_json`, :func:`read_html`, and :func:`read_xml` no longer accept raw string or byte representation of the data. That type of data must be wrapped in a :py:class:`StringIO` or :py:class:`BytesIO` (:issue:`53767`) +- :meth:`Series.dt.to_pydatetime` now returns a :class:`Series` of :py:class:`datetime.datetime` objects (:issue:`52459`) - All arguments except ``name`` in :meth:`Index.rename` are now keyword only (:issue:`56493`) - All arguments except the first ``path``-like argument in IO writers are now keyword only (:issue:`54229`) - All arguments in :meth:`Index.sort_values` are now keyword only (:issue:`56493`) @@ -120,6 +121,7 @@ Removal of prior version deprecations/changes - Enforced deprecation disallowing parsing datetimes with mixed time zones unless user passes ``utc=True`` to :func:`to_datetime` (:issue:`57275`) - Enforced silent-downcasting deprecation for :ref:`all relevant methods ` (:issue:`54710`) - In :meth:`DataFrame.stack`, the default value of ``future_stack`` is now ``True``; specifying ``False`` will raise a ``FutureWarning`` (:issue:`55448`) +- Passing both ``freq`` and ``fill_value`` in :meth:`DataFrame.shift` and :meth:`Series.shift` and :meth:`.DataFrameGroupBy.shift` now raises a ``ValueError`` (:issue:`54818`) - Removed :meth:`DateOffset.is_anchored` and :meth:`offsets.Tick.is_anchored` (:issue:`56594`) - Removed ``DataFrame.applymap``, ``Styler.applymap`` and ``Styler.applymap_index`` (:issue:`52364`) - Removed ``DataFrame.bool`` and ``Series.bool`` (:issue:`51756`) diff --git a/pandas/conftest.py b/pandas/conftest.py index 47316a4ba3526..acd802a27ea8c 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -158,10 +158,6 @@ def pytest_collection_modifyitems(items, config) -> None: ("SeriesGroupBy.idxmax", "The behavior of Series.idxmax"), # Docstring divides by zero to show behavior difference ("missing.mask_zero_div_zero", "divide by zero encountered"), - ( - "to_pydatetime", - "The behavior of DatetimeProperties.to_pydatetime is deprecated", - ), ( "pandas.core.generic.NDFrame.first", "first is deprecated and will be removed in a future version. " diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py index aeb6ef481e060..f4284cb0d0e5e 100644 --- a/pandas/core/arrays/arrow/array.py +++ b/pandas/core/arrays/arrow/array.py @@ -2918,7 +2918,9 @@ def _dt_month_name(self, locale: str | None = None) -> Self: locale = "C" return type(self)(pc.strftime(self._pa_array, format="%B", locale=locale)) - def _dt_to_pydatetime(self) -> np.ndarray: + def _dt_to_pydatetime(self) -> Series: + from pandas import Series + if pa.types.is_date(self.dtype.pyarrow_dtype): raise ValueError( f"to_pydatetime cannot be called with {self.dtype.pyarrow_dtype} type. " @@ -2927,7 +2929,7 @@ def _dt_to_pydatetime(self) -> np.ndarray: data = self._pa_array.to_pylist() if self._dtype.pyarrow_dtype.unit == "ns": data = [None if ts is None else ts.to_pydatetime(warn=False) for ts in data] - return np.array(data, dtype=object) + return Series(data, dtype=object) def _dt_tz_localize( self, diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 57f2cdef5457a..abb043361483c 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -5588,14 +5588,9 @@ def shift( ) -> DataFrame: if freq is not None and fill_value is not lib.no_default: # GH#53832 - warnings.warn( - "Passing a 'freq' together with a 'fill_value' silently ignores " - "the fill_value and is deprecated. This will raise in a future " - "version.", - FutureWarning, - stacklevel=find_stack_level(), + raise ValueError( + "Passing a 'freq' together with a 'fill_value' is not allowed." ) - fill_value = lib.no_default if self.empty: return self.copy() diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 7614951566bf9..e335929914101 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -10271,14 +10271,9 @@ def shift( if freq is not None and fill_value is not lib.no_default: # GH#53832 - warnings.warn( - "Passing a 'freq' together with a 'fill_value' silently ignores " - "the fill_value and is deprecated. This will raise in a future " - "version.", - FutureWarning, - stacklevel=find_stack_level(), + raise ValueError( + "Passing a 'freq' together with a 'fill_value' is not allowed." ) - fill_value = lib.no_default if periods == 0: return self.copy(deep=None) diff --git a/pandas/core/indexes/accessors.py b/pandas/core/indexes/accessors.py index 8a742a0a9d57d..5881f5e040370 100644 --- a/pandas/core/indexes/accessors.py +++ b/pandas/core/indexes/accessors.py @@ -8,12 +8,10 @@ NoReturn, cast, ) -import warnings import numpy as np from pandas._libs import lib -from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.common import ( is_integer_dtype, @@ -213,16 +211,8 @@ def _delegate_method(self, name: str, *args, **kwargs): def to_pytimedelta(self): return cast(ArrowExtensionArray, self._parent.array)._dt_to_pytimedelta() - def to_pydatetime(self): + def to_pydatetime(self) -> Series: # GH#20306 - warnings.warn( - f"The behavior of {type(self).__name__}.to_pydatetime is deprecated, " - "in a future version this will return a Series containing python " - "datetime objects instead of an ndarray. To retain the old behavior, " - "call `np.array` on the result", - FutureWarning, - stacklevel=find_stack_level(), - ) return cast(ArrowExtensionArray, self._parent.array)._dt_to_pydatetime() def isocalendar(self) -> DataFrame: @@ -318,15 +308,9 @@ class DatetimeProperties(Properties): Raises TypeError if the Series does not contain datetimelike values. """ - def to_pydatetime(self) -> np.ndarray: + def to_pydatetime(self) -> Series: """ - Return the data as an array of :class:`datetime.datetime` objects. - - .. deprecated:: 2.1.0 - - The current behavior of dt.to_pydatetime is deprecated. - In a future version this will return a Series containing python - datetime objects instead of a ndarray. + Return the data as a Series of :class:`datetime.datetime` objects. Timezone information is retained if present. @@ -353,8 +337,9 @@ def to_pydatetime(self) -> np.ndarray: dtype: datetime64[ns] >>> s.dt.to_pydatetime() - array([datetime.datetime(2018, 3, 10, 0, 0), - datetime.datetime(2018, 3, 11, 0, 0)], dtype=object) + 0 2018-03-10 00:00:00 + 1 2018-03-11 00:00:00 + dtype: object pandas' nanosecond precision is truncated to microseconds. @@ -365,19 +350,14 @@ def to_pydatetime(self) -> np.ndarray: dtype: datetime64[ns] >>> s.dt.to_pydatetime() - array([datetime.datetime(2018, 3, 10, 0, 0), - datetime.datetime(2018, 3, 10, 0, 0)], dtype=object) + 0 2018-03-10 00:00:00 + 1 2018-03-10 00:00:00 + dtype: object """ # GH#20306 - warnings.warn( - f"The behavior of {type(self).__name__}.to_pydatetime is deprecated, " - "in a future version this will return a Series containing python " - "datetime objects instead of an ndarray. To retain the old behavior, " - "call `np.array` on the result", - FutureWarning, - stacklevel=find_stack_level(), - ) - return self._get_values().to_pydatetime() + from pandas import Series + + return Series(self._get_values().to_pydatetime(), dtype=object) @property def freq(self): diff --git a/pandas/io/sql.py b/pandas/io/sql.py index d816bb2fc09d3..9024961688c6b 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1037,10 +1037,7 @@ def insert_data(self) -> tuple[list[str], list[np.ndarray]]: # GH#53854 to_pydatetime not supported for pyarrow date dtypes d = ser._values.to_numpy(dtype=object) else: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=FutureWarning) - # GH#52459 to_pydatetime will return Index[object] - d = np.asarray(ser.dt.to_pydatetime(), dtype=object) + d = ser.dt.to_pydatetime()._values else: d = ser._values.to_pydatetime() elif ser.dtype.kind == "m": diff --git a/pandas/tests/extension/test_arrow.py b/pandas/tests/extension/test_arrow.py index fe6e484fbd6e7..5d634c9aeb14f 100644 --- a/pandas/tests/extension/test_arrow.py +++ b/pandas/tests/extension/test_arrow.py @@ -2676,18 +2676,13 @@ def test_dt_to_pydatetime(): # GH 51859 data = [datetime(2022, 1, 1), datetime(2023, 1, 1)] ser = pd.Series(data, dtype=ArrowDtype(pa.timestamp("ns"))) + result = ser.dt.to_pydatetime() + expected = pd.Series(data, dtype=object) + tm.assert_series_equal(result, expected) + assert all(type(expected.iloc[i]) is datetime for i in range(len(expected))) - msg = "The behavior of ArrowTemporalProperties.to_pydatetime is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = ser.dt.to_pydatetime() - expected = np.array(data, dtype=object) - tm.assert_numpy_array_equal(result, expected) - assert all(type(res) is datetime for res in result) - - msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - expected = ser.astype("datetime64[ns]").dt.to_pydatetime() - tm.assert_numpy_array_equal(result, expected) + expected = ser.astype("datetime64[ns]").dt.to_pydatetime() + tm.assert_series_equal(result, expected) @pytest.mark.parametrize("date_type", [32, 64]) @@ -2697,10 +2692,8 @@ def test_dt_to_pydatetime_date_error(date_type): [date(2022, 12, 31)], dtype=ArrowDtype(getattr(pa, f"date{date_type}")()), ) - msg = "The behavior of ArrowTemporalProperties.to_pydatetime is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - with pytest.raises(ValueError, match="to_pydatetime cannot be called with"): - ser.dt.to_pydatetime() + with pytest.raises(ValueError, match="to_pydatetime cannot be called with"): + ser.dt.to_pydatetime() def test_dt_tz_localize_unsupported_tz_options(): diff --git a/pandas/tests/frame/methods/test_shift.py b/pandas/tests/frame/methods/test_shift.py index 5bcfaf8b199bf..72c1a123eac98 100644 --- a/pandas/tests/frame/methods/test_shift.py +++ b/pandas/tests/frame/methods/test_shift.py @@ -30,23 +30,20 @@ def test_shift_axis1_with_valid_fill_value_one_array(self): expected2 = DataFrame([12345] * 5, dtype="Float64") tm.assert_frame_equal(res2, expected2) - def test_shift_deprecate_freq_and_fill_value(self, frame_or_series): + def test_shift_disallow_freq_and_fill_value(self, frame_or_series): # Can't pass both! obj = frame_or_series( np.random.default_rng(2).standard_normal(5), index=date_range("1/1/2000", periods=5, freq="h"), ) - msg = ( - "Passing a 'freq' together with a 'fill_value' silently ignores the " - "fill_value" - ) - with tm.assert_produces_warning(FutureWarning, match=msg): + msg = "Passing a 'freq' together with a 'fill_value'" + with pytest.raises(ValueError, match=msg): obj.shift(1, fill_value=1, freq="h") if frame_or_series is DataFrame: obj.columns = date_range("1/1/2000", periods=1, freq="h") - with tm.assert_produces_warning(FutureWarning, match=msg): + with pytest.raises(ValueError, match=msg): obj.shift(1, axis=1, fill_value=1, freq="h") @pytest.mark.parametrize( @@ -717,13 +714,6 @@ def test_shift_with_iterable_freq_and_fill_value(self): df.shift(1, freq="h"), ) - msg = ( - "Passing a 'freq' together with a 'fill_value' silently ignores the " - "fill_value" - ) - with tm.assert_produces_warning(FutureWarning, match=msg): - df.shift([1, 2], fill_value=1, freq="h") - def test_shift_with_iterable_check_other_arguments(self): # GH#44424 data = {"a": [1, 2], "b": [4, 5]} diff --git a/pandas/tests/groupby/methods/test_groupby_shift_diff.py b/pandas/tests/groupby/methods/test_groupby_shift_diff.py index 41e0ee93a5941..1256046d81949 100644 --- a/pandas/tests/groupby/methods/test_groupby_shift_diff.py +++ b/pandas/tests/groupby/methods/test_groupby_shift_diff.py @@ -167,14 +167,12 @@ def test_shift_periods_freq(): tm.assert_frame_equal(result, expected) -def test_shift_deprecate_freq_and_fill_value(): +def test_shift_disallow_freq_and_fill_value(): # GH 53832 data = {"a": [1, 2, 3, 4, 5, 6], "b": [0, 0, 0, 1, 1, 1]} df = DataFrame(data, index=date_range(start="20100101", periods=6)) - msg = ( - "Passing a 'freq' together with a 'fill_value' silently ignores the fill_value" - ) - with tm.assert_produces_warning(FutureWarning, match=msg): + msg = "Passing a 'freq' together with a 'fill_value'" + with pytest.raises(ValueError, match=msg): df.groupby(df.index).shift(periods=-2, freq="D", fill_value="1") @@ -247,9 +245,6 @@ def test_group_shift_with_multiple_periods_and_both_fill_and_freq_deprecated(): {"a": [1, 2, 3, 4, 5], "b": [True, True, False, False, True]}, index=date_range("1/1/2000", periods=5, freq="h"), ) - msg = ( - "Passing a 'freq' together with a 'fill_value' silently ignores the " - "fill_value" - ) - with tm.assert_produces_warning(FutureWarning, match=msg): + msg = "Passing a 'freq' together with a 'fill_value'" + with pytest.raises(ValueError, match=msg): df.groupby("b")[["a"]].shift([1, 2], fill_value=1, freq="h") diff --git a/pandas/tests/series/accessors/test_cat_accessor.py b/pandas/tests/series/accessors/test_cat_accessor.py index 2d50b0f36904a..ca2768efd5c68 100644 --- a/pandas/tests/series/accessors/test_cat_accessor.py +++ b/pandas/tests/series/accessors/test_cat_accessor.py @@ -200,9 +200,6 @@ def test_dt_accessor_api_for_categorical(self, idx): if func == "to_period" and getattr(idx, "tz", None) is not None: # dropping TZ warn_cls.append(UserWarning) - if func == "to_pydatetime": - # deprecated to return Index[object] - warn_cls.append(FutureWarning) if warn_cls: warn_cls = tuple(warn_cls) else: diff --git a/pandas/tests/series/accessors/test_dt_accessor.py b/pandas/tests/series/accessors/test_dt_accessor.py index 17492f17132fd..5f0057ac50b47 100644 --- a/pandas/tests/series/accessors/test_dt_accessor.py +++ b/pandas/tests/series/accessors/test_dt_accessor.py @@ -114,10 +114,8 @@ def test_dt_namespace_accessor_datetime64(self, freq): for prop in ok_for_dt_methods: getattr(ser.dt, prop) - msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = ser.dt.to_pydatetime() - assert isinstance(result, np.ndarray) + result = ser.dt.to_pydatetime() + assert isinstance(result, Series) assert result.dtype == object result = ser.dt.tz_localize("US/Eastern") @@ -153,10 +151,8 @@ def test_dt_namespace_accessor_datetime64tz(self): for prop in ok_for_dt_methods: getattr(ser.dt, prop) - msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated" - with tm.assert_produces_warning(FutureWarning, match=msg): - result = ser.dt.to_pydatetime() - assert isinstance(result, np.ndarray) + result = ser.dt.to_pydatetime() + assert isinstance(result, Series) assert result.dtype == object result = ser.dt.tz_convert("CET")