CLN: shift with freq and fill_value, to_pydatetime returning Series (pandas-dev#57425)

* Enforce shift with freq and fill_value

* Enforce to_pydatetime change

* Change doc example

* Use to_numpy

* Use _values

* Fix doctest

Author: mroeschke

doc/source/whatsnew/v3.0.0.rst

@@ -112,6 +112,7 @@ Deprecations
 Removal of prior version deprecations/changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 - :func:`read_excel`, :func:`read_json`, :func:`read_html`, and :func:`read_xml` no longer accept raw string or byte representation of the data. That type of data must be wrapped in a :py:class:`StringIO` or :py:class:`BytesIO` (:issue:`53767`)
+- :meth:`Series.dt.to_pydatetime` now returns a :class:`Series` of :py:class:`datetime.datetime` objects (:issue:`52459`)
 - All arguments except ``name`` in :meth:`Index.rename` are now keyword only (:issue:`56493`)
 - All arguments except the first ``path``-like argument in IO writers are now keyword only (:issue:`54229`)
 - All arguments in :meth:`Index.sort_values` are now keyword only (:issue:`56493`)
@@ -120,6 +121,7 @@ Removal of prior version deprecations/changes
 - Enforced deprecation disallowing parsing datetimes with mixed time zones unless user passes ``utc=True`` to :func:`to_datetime` (:issue:`57275`)
 - Enforced silent-downcasting deprecation for :ref:`all relevant methods <whatsnew_220.silent_downcasting>` (:issue:`54710`)
 - In :meth:`DataFrame.stack`, the default value of ``future_stack`` is now ``True``; specifying ``False`` will raise a ``FutureWarning`` (:issue:`55448`)
+- Passing both ``freq`` and ``fill_value`` in :meth:`DataFrame.shift` and :meth:`Series.shift` and :meth:`.DataFrameGroupBy.shift` now raises a ``ValueError`` (:issue:`54818`)
 - Removed :meth:`DateOffset.is_anchored` and :meth:`offsets.Tick.is_anchored` (:issue:`56594`)
 - Removed ``DataFrame.applymap``, ``Styler.applymap`` and ``Styler.applymap_index`` (:issue:`52364`)
 - Removed ``DataFrame.bool`` and ``Series.bool`` (:issue:`51756`)

pandas/conftest.py

@@ -158,10 +158,6 @@ def pytest_collection_modifyitems(items, config) -> None:
         ("SeriesGroupBy.idxmax", "The behavior of Series.idxmax"),
         # Docstring divides by zero to show behavior difference
         ("missing.mask_zero_div_zero", "divide by zero encountered"),
-        (
-            "to_pydatetime",
-            "The behavior of DatetimeProperties.to_pydatetime is deprecated",
-        ),
         (
             "pandas.core.generic.NDFrame.first",
             "first is deprecated and will be removed in a future version. "

pandas/core/arrays/arrow/array.py

@@ -2918,7 +2918,9 @@ def _dt_month_name(self, locale: str | None = None) -> Self:
             locale = "C"
         return type(self)(pc.strftime(self._pa_array, format="%B", locale=locale))
 
-    def _dt_to_pydatetime(self) -> np.ndarray:
+    def _dt_to_pydatetime(self) -> Series:
+        from pandas import Series
+
         if pa.types.is_date(self.dtype.pyarrow_dtype):
             raise ValueError(
                 f"to_pydatetime cannot be called with {self.dtype.pyarrow_dtype} type. "
@@ -2927,7 +2929,7 @@ def _dt_to_pydatetime(self) -> np.ndarray:
         data = self._pa_array.to_pylist()
         if self._dtype.pyarrow_dtype.unit == "ns":
             data = [None if ts is None else ts.to_pydatetime(warn=False) for ts in data]
-        return np.array(data, dtype=object)
+        return Series(data, dtype=object)
 
     def _dt_tz_localize(
         self,

pandas/core/frame.py

@@ -5588,14 +5588,9 @@ def shift(
     ) -> DataFrame:
         if freq is not None and fill_value is not lib.no_default:
             # GH#53832
-            warnings.warn(
-                "Passing a 'freq' together with a 'fill_value' silently ignores "
-                "the fill_value and is deprecated. This will raise in a future "
-                "version.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
+            raise ValueError(
+                "Passing a 'freq' together with a 'fill_value' is not allowed."
             )
-            fill_value = lib.no_default
 
         if self.empty:
             return self.copy()

pandas/core/generic.py

@@ -10269,14 +10269,9 @@ def shift(
 
         if freq is not None and fill_value is not lib.no_default:
             # GH#53832
-            warnings.warn(
-                "Passing a 'freq' together with a 'fill_value' silently ignores "
-                "the fill_value and is deprecated. This will raise in a future "
-                "version.",
-                FutureWarning,
-                stacklevel=find_stack_level(),
+            raise ValueError(
+                "Passing a 'freq' together with a 'fill_value' is not allowed."
             )
-            fill_value = lib.no_default
 
         if periods == 0:
             return self.copy(deep=False)

pandas/core/indexes/accessors.py

@@ -8,12 +8,10 @@
     NoReturn,
     cast,
 )
-import warnings
 
 import numpy as np
 
 from pandas._libs import lib
-from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import (
     is_integer_dtype,
@@ -213,16 +211,8 @@ def _delegate_method(self, name: str, *args, **kwargs):
     def to_pytimedelta(self):
         return cast(ArrowExtensionArray, self._parent.array)._dt_to_pytimedelta()
 
-    def to_pydatetime(self):
+    def to_pydatetime(self) -> Series:
         # GH#20306
-        warnings.warn(
-            f"The behavior of {type(self).__name__}.to_pydatetime is deprecated, "
-            "in a future version this will return a Series containing python "
-            "datetime objects instead of an ndarray. To retain the old behavior, "
-            "call `np.array` on the result",
-            FutureWarning,
-            stacklevel=find_stack_level(),
-        )
         return cast(ArrowExtensionArray, self._parent.array)._dt_to_pydatetime()
 
     def isocalendar(self) -> DataFrame:
@@ -318,15 +308,9 @@ class DatetimeProperties(Properties):
     Raises TypeError if the Series does not contain datetimelike values.
     """
 
-    def to_pydatetime(self) -> np.ndarray:
+    def to_pydatetime(self) -> Series:
         """
-        Return the data as an array of :class:`datetime.datetime` objects.
-
-        .. deprecated:: 2.1.0
-
-            The current behavior of dt.to_pydatetime is deprecated.
-            In a future version this will return a Series containing python
-            datetime objects instead of a ndarray.
+        Return the data as a Series of :class:`datetime.datetime` objects.
 
         Timezone information is retained if present.
 
@@ -353,8 +337,9 @@ def to_pydatetime(self) -> np.ndarray:
         dtype: datetime64[ns]
 
         >>> s.dt.to_pydatetime()
-        array([datetime.datetime(2018, 3, 10, 0, 0),
-               datetime.datetime(2018, 3, 11, 0, 0)], dtype=object)
+        0    2018-03-10 00:00:00
+        1    2018-03-11 00:00:00
+        dtype: object
 
         pandas' nanosecond precision is truncated to microseconds.
 
@@ -365,19 +350,14 @@ def to_pydatetime(self) -> np.ndarray:
         dtype: datetime64[ns]
 
         >>> s.dt.to_pydatetime()
-        array([datetime.datetime(2018, 3, 10, 0, 0),
-               datetime.datetime(2018, 3, 10, 0, 0)], dtype=object)
+        0    2018-03-10 00:00:00
+        1    2018-03-10 00:00:00
+        dtype: object
         """
         # GH#20306
-        warnings.warn(
-            f"The behavior of {type(self).__name__}.to_pydatetime is deprecated, "
-            "in a future version this will return a Series containing python "
-            "datetime objects instead of an ndarray. To retain the old behavior, "
-            "call `np.array` on the result",
-            FutureWarning,
-            stacklevel=find_stack_level(),
-        )
-        return self._get_values().to_pydatetime()
+        from pandas import Series
+
+        return Series(self._get_values().to_pydatetime(), dtype=object)
 
     @property
     def freq(self):

pandas/io/sql.py

@@ -1037,10 +1037,7 @@ def insert_data(self) -> tuple[list[str], list[np.ndarray]]:
                         # GH#53854 to_pydatetime not supported for pyarrow date dtypes
                         d = ser._values.to_numpy(dtype=object)
                     else:
-                        with warnings.catch_warnings():
-                            warnings.filterwarnings("ignore", category=FutureWarning)
-                            # GH#52459 to_pydatetime will return Index[object]
-                            d = np.asarray(ser.dt.to_pydatetime(), dtype=object)
+                        d = ser.dt.to_pydatetime()._values
                 else:
                     d = ser._values.to_pydatetime()
             elif ser.dtype.kind == "m":

pandas/tests/extension/test_arrow.py

@@ -2676,18 +2676,13 @@ def test_dt_to_pydatetime():
     # GH 51859
     data = [datetime(2022, 1, 1), datetime(2023, 1, 1)]
     ser = pd.Series(data, dtype=ArrowDtype(pa.timestamp("ns")))
+    result = ser.dt.to_pydatetime()
+    expected = pd.Series(data, dtype=object)
+    tm.assert_series_equal(result, expected)
+    assert all(type(expected.iloc[i]) is datetime for i in range(len(expected)))
 
-    msg = "The behavior of ArrowTemporalProperties.to_pydatetime is deprecated"
-    with tm.assert_produces_warning(FutureWarning, match=msg):
-        result = ser.dt.to_pydatetime()
-    expected = np.array(data, dtype=object)
-    tm.assert_numpy_array_equal(result, expected)
-    assert all(type(res) is datetime for res in result)
-
-    msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated"
-    with tm.assert_produces_warning(FutureWarning, match=msg):
-        expected = ser.astype("datetime64[ns]").dt.to_pydatetime()
-    tm.assert_numpy_array_equal(result, expected)
+    expected = ser.astype("datetime64[ns]").dt.to_pydatetime()
+    tm.assert_series_equal(result, expected)
 
 
 @pytest.mark.parametrize("date_type", [32, 64])
@@ -2697,10 +2692,8 @@ def test_dt_to_pydatetime_date_error(date_type):
         [date(2022, 12, 31)],
         dtype=ArrowDtype(getattr(pa, f"date{date_type}")()),
     )
-    msg = "The behavior of ArrowTemporalProperties.to_pydatetime is deprecated"
-    with tm.assert_produces_warning(FutureWarning, match=msg):
-        with pytest.raises(ValueError, match="to_pydatetime cannot be called with"):
-            ser.dt.to_pydatetime()
+    with pytest.raises(ValueError, match="to_pydatetime cannot be called with"):
+        ser.dt.to_pydatetime()
 
 
 def test_dt_tz_localize_unsupported_tz_options():

pandas/tests/frame/methods/test_shift.py

@@ -30,23 +30,20 @@ def test_shift_axis1_with_valid_fill_value_one_array(self):
         expected2 = DataFrame([12345] * 5, dtype="Float64")
         tm.assert_frame_equal(res2, expected2)
 
-    def test_shift_deprecate_freq_and_fill_value(self, frame_or_series):
+    def test_shift_disallow_freq_and_fill_value(self, frame_or_series):
         # Can't pass both!
         obj = frame_or_series(
             np.random.default_rng(2).standard_normal(5),
             index=date_range("1/1/2000", periods=5, freq="h"),
         )
 
-        msg = (
-            "Passing a 'freq' together with a 'fill_value' silently ignores the "
-            "fill_value"
-        )
-        with tm.assert_produces_warning(FutureWarning, match=msg):
+        msg = "Passing a 'freq' together with a 'fill_value'"
+        with pytest.raises(ValueError, match=msg):
             obj.shift(1, fill_value=1, freq="h")
 
         if frame_or_series is DataFrame:
             obj.columns = date_range("1/1/2000", periods=1, freq="h")
-            with tm.assert_produces_warning(FutureWarning, match=msg):
+            with pytest.raises(ValueError, match=msg):
                 obj.shift(1, axis=1, fill_value=1, freq="h")
 
     @pytest.mark.parametrize(
@@ -717,13 +714,6 @@ def test_shift_with_iterable_freq_and_fill_value(self):
             df.shift(1, freq="h"),
         )
 
-        msg = (
-            "Passing a 'freq' together with a 'fill_value' silently ignores the "
-            "fill_value"
-        )
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            df.shift([1, 2], fill_value=1, freq="h")
-
     def test_shift_with_iterable_check_other_arguments(self):
         # GH#44424
         data = {"a": [1, 2], "b": [4, 5]}

pandas/tests/groupby/methods/test_groupby_shift_diff.py

@@ -167,14 +167,12 @@ def test_shift_periods_freq():
     tm.assert_frame_equal(result, expected)
 
 
-def test_shift_deprecate_freq_and_fill_value():
+def test_shift_disallow_freq_and_fill_value():
     # GH 53832
     data = {"a": [1, 2, 3, 4, 5, 6], "b": [0, 0, 0, 1, 1, 1]}
     df = DataFrame(data, index=date_range(start="20100101", periods=6))
-    msg = (
-        "Passing a 'freq' together with a 'fill_value' silently ignores the fill_value"
-    )
-    with tm.assert_produces_warning(FutureWarning, match=msg):
+    msg = "Passing a 'freq' together with a 'fill_value'"
+    with pytest.raises(ValueError, match=msg):
         df.groupby(df.index).shift(periods=-2, freq="D", fill_value="1")
 
 
@@ -247,9 +245,6 @@ def test_group_shift_with_multiple_periods_and_both_fill_and_freq_deprecated():
         {"a": [1, 2, 3, 4, 5], "b": [True, True, False, False, True]},
         index=date_range("1/1/2000", periods=5, freq="h"),
     )
-    msg = (
-        "Passing a 'freq' together with a 'fill_value' silently ignores the "
-        "fill_value"
-    )
-    with tm.assert_produces_warning(FutureWarning, match=msg):
+    msg = "Passing a 'freq' together with a 'fill_value'"
+    with pytest.raises(ValueError, match=msg):
         df.groupby("b")[["a"]].shift([1, 2], fill_value=1, freq="h")

pandas/tests/series/accessors/test_cat_accessor.py

@@ -200,9 +200,6 @@ def test_dt_accessor_api_for_categorical(self, idx):
             if func == "to_period" and getattr(idx, "tz", None) is not None:
                 # dropping TZ
                 warn_cls.append(UserWarning)
-            if func == "to_pydatetime":
-                # deprecated to return Index[object]
-                warn_cls.append(FutureWarning)
             if warn_cls:
                 warn_cls = tuple(warn_cls)
             else:

pandas/tests/series/accessors/test_dt_accessor.py

@@ -114,10 +114,8 @@ def test_dt_namespace_accessor_datetime64(self, freq):
         for prop in ok_for_dt_methods:
             getattr(ser.dt, prop)
 
-        msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = ser.dt.to_pydatetime()
-        assert isinstance(result, np.ndarray)
+        result = ser.dt.to_pydatetime()
+        assert isinstance(result, Series)
         assert result.dtype == object
 
         result = ser.dt.tz_localize("US/Eastern")
@@ -153,10 +151,8 @@ def test_dt_namespace_accessor_datetime64tz(self):
         for prop in ok_for_dt_methods:
             getattr(ser.dt, prop)
 
-        msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            result = ser.dt.to_pydatetime()
-        assert isinstance(result, np.ndarray)
+        result = ser.dt.to_pydatetime()
+        assert isinstance(result, Series)
         assert result.dtype == object
 
         result = ser.dt.tz_convert("CET")