Skip to content

CLN: shift with freq and fill_value, to_pydatetime returning Series #57425

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Feb 20, 2024
2 changes: 2 additions & 0 deletions doc/source/whatsnew/v3.0.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ Deprecations
Removal of prior version deprecations/changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- :func:`read_excel`, :func:`read_json`, :func:`read_html`, and :func:`read_xml` no longer accept raw string or byte representation of the data. That type of data must be wrapped in a :py:class:`StringIO` or :py:class:`BytesIO` (:issue:`53767`)
- :meth:`Series.dt.to_pydatetime` now returns a :class:`Series` of :py:class:`datetime.datetime` objects (:issue:`52459`)
- All arguments except ``name`` in :meth:`Index.rename` are now keyword only (:issue:`56493`)
- All arguments except the first ``path``-like argument in IO writers are now keyword only (:issue:`54229`)
- All arguments in :meth:`Index.sort_values` are now keyword only (:issue:`56493`)
Expand All @@ -120,6 +121,7 @@ Removal of prior version deprecations/changes
- Enforced deprecation disallowing parsing datetimes with mixed time zones unless user passes ``utc=True`` to :func:`to_datetime` (:issue:`57275`)
- Enforced silent-downcasting deprecation for :ref:`all relevant methods <whatsnew_220.silent_downcasting>` (:issue:`54710`)
- In :meth:`DataFrame.stack`, the default value of ``future_stack`` is now ``True``; specifying ``False`` will raise a ``FutureWarning`` (:issue:`55448`)
- Passing both ``freq`` and ``fill_value`` in :meth:`DataFrame.shift` and :meth:`Series.shift` and :meth:`.DataFrameGroupBy.shift` now raises a ``ValueError`` (:issue:`54818`)
- Removed :meth:`DateOffset.is_anchored` and :meth:`offsets.Tick.is_anchored` (:issue:`56594`)
- Removed ``DataFrame.applymap``, ``Styler.applymap`` and ``Styler.applymap_index`` (:issue:`52364`)
- Removed ``DataFrame.bool`` and ``Series.bool`` (:issue:`51756`)
Expand Down
4 changes: 0 additions & 4 deletions pandas/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,10 +158,6 @@ def pytest_collection_modifyitems(items, config) -> None:
("SeriesGroupBy.idxmax", "The behavior of Series.idxmax"),
# Docstring divides by zero to show behavior difference
("missing.mask_zero_div_zero", "divide by zero encountered"),
(
"to_pydatetime",
"The behavior of DatetimeProperties.to_pydatetime is deprecated",
),
(
"pandas.core.generic.NDFrame.first",
"first is deprecated and will be removed in a future version. "
Expand Down
6 changes: 4 additions & 2 deletions pandas/core/arrays/arrow/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -2918,7 +2918,9 @@ def _dt_month_name(self, locale: str | None = None) -> Self:
locale = "C"
return type(self)(pc.strftime(self._pa_array, format="%B", locale=locale))

def _dt_to_pydatetime(self) -> np.ndarray:
def _dt_to_pydatetime(self) -> Series:
from pandas import Series

if pa.types.is_date(self.dtype.pyarrow_dtype):
raise ValueError(
f"to_pydatetime cannot be called with {self.dtype.pyarrow_dtype} type. "
Expand All @@ -2927,7 +2929,7 @@ def _dt_to_pydatetime(self) -> np.ndarray:
data = self._pa_array.to_pylist()
if self._dtype.pyarrow_dtype.unit == "ns":
data = [None if ts is None else ts.to_pydatetime(warn=False) for ts in data]
return np.array(data, dtype=object)
return Series(data, dtype=object)

def _dt_tz_localize(
self,
Expand Down
9 changes: 2 additions & 7 deletions pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -5588,14 +5588,9 @@ def shift(
) -> DataFrame:
if freq is not None and fill_value is not lib.no_default:
# GH#53832
warnings.warn(
"Passing a 'freq' together with a 'fill_value' silently ignores "
"the fill_value and is deprecated. This will raise in a future "
"version.",
FutureWarning,
stacklevel=find_stack_level(),
raise ValueError(
"Passing a 'freq' together with a 'fill_value' is not allowed."
)
fill_value = lib.no_default

if self.empty:
return self.copy()
Expand Down
9 changes: 2 additions & 7 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -10271,14 +10271,9 @@ def shift(

if freq is not None and fill_value is not lib.no_default:
# GH#53832
warnings.warn(
"Passing a 'freq' together with a 'fill_value' silently ignores "
"the fill_value and is deprecated. This will raise in a future "
"version.",
FutureWarning,
stacklevel=find_stack_level(),
raise ValueError(
"Passing a 'freq' together with a 'fill_value' is not allowed."
)
fill_value = lib.no_default

if periods == 0:
return self.copy(deep=None)
Expand Down
44 changes: 12 additions & 32 deletions pandas/core/indexes/accessors.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,10 @@
NoReturn,
cast,
)
import warnings

import numpy as np

from pandas._libs import lib
from pandas.util._exceptions import find_stack_level

from pandas.core.dtypes.common import (
is_integer_dtype,
Expand Down Expand Up @@ -213,16 +211,8 @@ def _delegate_method(self, name: str, *args, **kwargs):
def to_pytimedelta(self):
return cast(ArrowExtensionArray, self._parent.array)._dt_to_pytimedelta()

def to_pydatetime(self):
def to_pydatetime(self) -> Series:
# GH#20306
warnings.warn(
f"The behavior of {type(self).__name__}.to_pydatetime is deprecated, "
"in a future version this will return a Series containing python "
"datetime objects instead of an ndarray. To retain the old behavior, "
"call `np.array` on the result",
FutureWarning,
stacklevel=find_stack_level(),
)
return cast(ArrowExtensionArray, self._parent.array)._dt_to_pydatetime()

def isocalendar(self) -> DataFrame:
Expand Down Expand Up @@ -318,15 +308,9 @@ class DatetimeProperties(Properties):
Raises TypeError if the Series does not contain datetimelike values.
"""

def to_pydatetime(self) -> np.ndarray:
def to_pydatetime(self) -> Series:
"""
Return the data as an array of :class:`datetime.datetime` objects.

.. deprecated:: 2.1.0

The current behavior of dt.to_pydatetime is deprecated.
In a future version this will return a Series containing python
datetime objects instead of a ndarray.
Return the data as a Series of :class:`datetime.datetime` objects.

Timezone information is retained if present.

Expand All @@ -353,8 +337,9 @@ def to_pydatetime(self) -> np.ndarray:
dtype: datetime64[ns]

>>> s.dt.to_pydatetime()
array([datetime.datetime(2018, 3, 10, 0, 0),
datetime.datetime(2018, 3, 11, 0, 0)], dtype=object)
0 2018-03-10 00:00:00
1 2018-03-11 00:00:00
dtype: object

pandas' nanosecond precision is truncated to microseconds.

Expand All @@ -365,19 +350,14 @@ def to_pydatetime(self) -> np.ndarray:
dtype: datetime64[ns]

>>> s.dt.to_pydatetime()
array([datetime.datetime(2018, 3, 10, 0, 0),
datetime.datetime(2018, 3, 10, 0, 0)], dtype=object)
0 2018-03-10 00:00:00
1 2018-03-10 00:00:00
dtype: object
"""
# GH#20306
warnings.warn(
f"The behavior of {type(self).__name__}.to_pydatetime is deprecated, "
"in a future version this will return a Series containing python "
"datetime objects instead of an ndarray. To retain the old behavior, "
"call `np.array` on the result",
FutureWarning,
stacklevel=find_stack_level(),
)
return self._get_values().to_pydatetime()
from pandas import Series

return Series(self._get_values().to_pydatetime(), dtype=object)

@property
def freq(self):
Expand Down
5 changes: 1 addition & 4 deletions pandas/io/sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -1037,10 +1037,7 @@ def insert_data(self) -> tuple[list[str], list[np.ndarray]]:
# GH#53854 to_pydatetime not supported for pyarrow date dtypes
d = ser._values.to_numpy(dtype=object)
else:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
# GH#52459 to_pydatetime will return Index[object]
d = np.asarray(ser.dt.to_pydatetime(), dtype=object)
d = ser.dt.to_pydatetime()._values
else:
d = ser._values.to_pydatetime()
elif ser.dtype.kind == "m":
Expand Down
23 changes: 8 additions & 15 deletions pandas/tests/extension/test_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2676,18 +2676,13 @@ def test_dt_to_pydatetime():
# GH 51859
data = [datetime(2022, 1, 1), datetime(2023, 1, 1)]
ser = pd.Series(data, dtype=ArrowDtype(pa.timestamp("ns")))
result = ser.dt.to_pydatetime()
expected = pd.Series(data, dtype=object)
tm.assert_series_equal(result, expected)
assert all(type(expected.iloc[i]) is datetime for i in range(len(expected)))

msg = "The behavior of ArrowTemporalProperties.to_pydatetime is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = ser.dt.to_pydatetime()
expected = np.array(data, dtype=object)
tm.assert_numpy_array_equal(result, expected)
assert all(type(res) is datetime for res in result)

msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
expected = ser.astype("datetime64[ns]").dt.to_pydatetime()
tm.assert_numpy_array_equal(result, expected)
expected = ser.astype("datetime64[ns]").dt.to_pydatetime()
tm.assert_series_equal(result, expected)


@pytest.mark.parametrize("date_type", [32, 64])
Expand All @@ -2697,10 +2692,8 @@ def test_dt_to_pydatetime_date_error(date_type):
[date(2022, 12, 31)],
dtype=ArrowDtype(getattr(pa, f"date{date_type}")()),
)
msg = "The behavior of ArrowTemporalProperties.to_pydatetime is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
with pytest.raises(ValueError, match="to_pydatetime cannot be called with"):
ser.dt.to_pydatetime()
with pytest.raises(ValueError, match="to_pydatetime cannot be called with"):
ser.dt.to_pydatetime()


def test_dt_tz_localize_unsupported_tz_options():
Expand Down
18 changes: 4 additions & 14 deletions pandas/tests/frame/methods/test_shift.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,23 +30,20 @@ def test_shift_axis1_with_valid_fill_value_one_array(self):
expected2 = DataFrame([12345] * 5, dtype="Float64")
tm.assert_frame_equal(res2, expected2)

def test_shift_deprecate_freq_and_fill_value(self, frame_or_series):
def test_shift_disallow_freq_and_fill_value(self, frame_or_series):
# Can't pass both!
obj = frame_or_series(
np.random.default_rng(2).standard_normal(5),
index=date_range("1/1/2000", periods=5, freq="h"),
)

msg = (
"Passing a 'freq' together with a 'fill_value' silently ignores the "
"fill_value"
)
with tm.assert_produces_warning(FutureWarning, match=msg):
msg = "Passing a 'freq' together with a 'fill_value'"
with pytest.raises(ValueError, match=msg):
obj.shift(1, fill_value=1, freq="h")

if frame_or_series is DataFrame:
obj.columns = date_range("1/1/2000", periods=1, freq="h")
with tm.assert_produces_warning(FutureWarning, match=msg):
with pytest.raises(ValueError, match=msg):
obj.shift(1, axis=1, fill_value=1, freq="h")

@pytest.mark.parametrize(
Expand Down Expand Up @@ -717,13 +714,6 @@ def test_shift_with_iterable_freq_and_fill_value(self):
df.shift(1, freq="h"),
)

msg = (
"Passing a 'freq' together with a 'fill_value' silently ignores the "
"fill_value"
)
with tm.assert_produces_warning(FutureWarning, match=msg):
df.shift([1, 2], fill_value=1, freq="h")

def test_shift_with_iterable_check_other_arguments(self):
# GH#44424
data = {"a": [1, 2], "b": [4, 5]}
Expand Down
15 changes: 5 additions & 10 deletions pandas/tests/groupby/methods/test_groupby_shift_diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,14 +167,12 @@ def test_shift_periods_freq():
tm.assert_frame_equal(result, expected)


def test_shift_deprecate_freq_and_fill_value():
def test_shift_disallow_freq_and_fill_value():
# GH 53832
data = {"a": [1, 2, 3, 4, 5, 6], "b": [0, 0, 0, 1, 1, 1]}
df = DataFrame(data, index=date_range(start="20100101", periods=6))
msg = (
"Passing a 'freq' together with a 'fill_value' silently ignores the fill_value"
)
with tm.assert_produces_warning(FutureWarning, match=msg):
msg = "Passing a 'freq' together with a 'fill_value'"
with pytest.raises(ValueError, match=msg):
df.groupby(df.index).shift(periods=-2, freq="D", fill_value="1")


Expand Down Expand Up @@ -247,9 +245,6 @@ def test_group_shift_with_multiple_periods_and_both_fill_and_freq_deprecated():
{"a": [1, 2, 3, 4, 5], "b": [True, True, False, False, True]},
index=date_range("1/1/2000", periods=5, freq="h"),
)
msg = (
"Passing a 'freq' together with a 'fill_value' silently ignores the "
"fill_value"
)
with tm.assert_produces_warning(FutureWarning, match=msg):
msg = "Passing a 'freq' together with a 'fill_value'"
with pytest.raises(ValueError, match=msg):
df.groupby("b")[["a"]].shift([1, 2], fill_value=1, freq="h")
3 changes: 0 additions & 3 deletions pandas/tests/series/accessors/test_cat_accessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,6 @@ def test_dt_accessor_api_for_categorical(self, idx):
if func == "to_period" and getattr(idx, "tz", None) is not None:
# dropping TZ
warn_cls.append(UserWarning)
if func == "to_pydatetime":
# deprecated to return Index[object]
warn_cls.append(FutureWarning)
if warn_cls:
warn_cls = tuple(warn_cls)
else:
Expand Down
12 changes: 4 additions & 8 deletions pandas/tests/series/accessors/test_dt_accessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,10 +114,8 @@ def test_dt_namespace_accessor_datetime64(self, freq):
for prop in ok_for_dt_methods:
getattr(ser.dt, prop)

msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = ser.dt.to_pydatetime()
assert isinstance(result, np.ndarray)
result = ser.dt.to_pydatetime()
assert isinstance(result, Series)
assert result.dtype == object

result = ser.dt.tz_localize("US/Eastern")
Expand Down Expand Up @@ -153,10 +151,8 @@ def test_dt_namespace_accessor_datetime64tz(self):
for prop in ok_for_dt_methods:
getattr(ser.dt, prop)

msg = "The behavior of DatetimeProperties.to_pydatetime is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = ser.dt.to_pydatetime()
assert isinstance(result, np.ndarray)
result = ser.dt.to_pydatetime()
assert isinstance(result, Series)
assert result.dtype == object

result = ser.dt.tz_convert("CET")
Expand Down