Skip to content

Commit 09bdcbb

Browse files
authored
DEPR: Deprecate tshift and integrate it to shift (#34545)
1 parent 718bc9f commit 09bdcbb

File tree

10 files changed

+239
-81
lines changed

10 files changed

+239
-81
lines changed

doc/source/user_guide/timeseries.rst

+8-12
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,7 @@ The ``DatetimeIndex`` class contains many time series related optimizations:
516516
* A large range of dates for various offsets are pre-computed and cached
517517
under the hood in order to make generating subsequent date ranges very fast
518518
(just have to grab a slice).
519-
* Fast shifting using the ``shift`` and ``tshift`` method on pandas objects.
519+
* Fast shifting using the ``shift`` method on pandas objects.
520520
* Unioning of overlapping ``DatetimeIndex`` objects with the same frequency is
521521
very fast (important for fast data alignment).
522522
* Quick access to date fields via properties such as ``year``, ``month``, etc.
@@ -1462,23 +1462,19 @@ the pandas objects.
14621462
14631463
The ``shift`` method accepts an ``freq`` argument which can accept a
14641464
``DateOffset`` class or other ``timedelta``-like object or also an
1465-
:ref:`offset alias <timeseries.offset_aliases>`:
1465+
:ref:`offset alias <timeseries.offset_aliases>`.
1466+
1467+
When ``freq`` is specified, ``shift`` method changes all the dates in the index
1468+
rather than changing the alignment of the data and the index:
14661469

14671470
.. ipython:: python
14681471
1472+
ts.shift(5, freq='D')
14691473
ts.shift(5, freq=pd.offsets.BDay())
14701474
ts.shift(5, freq='BM')
14711475
1472-
Rather than changing the alignment of the data and the index, ``DataFrame`` and
1473-
``Series`` objects also have a :meth:`~Series.tshift` convenience method that
1474-
changes all the dates in the index by a specified number of offsets:
1475-
1476-
.. ipython:: python
1477-
1478-
ts.tshift(5, freq='D')
1479-
1480-
Note that with ``tshift``, the leading entry is no longer NaN because the data
1481-
is not being realigned.
1476+
Note that with when ``freq`` is specified, the leading entry is no longer NaN
1477+
because the data is not being realigned.
14821478

14831479
Frequency conversion
14841480
~~~~~~~~~~~~~~~~~~~~

doc/source/whatsnew/v1.1.0.rst

+1
Original file line numberDiff line numberDiff line change
@@ -767,6 +767,7 @@ Deprecations
767767
- :meth:`DatetimeIndex.week` and `DatetimeIndex.weekofyear` are deprecated and will be removed in a future version, use :meth:`DatetimeIndex.isocalendar().week` instead (:issue:`33595`)
768768
- :meth:`DatetimeArray.week` and `DatetimeArray.weekofyear` are deprecated and will be removed in a future version, use :meth:`DatetimeArray.isocalendar().week` instead (:issue:`33595`)
769769
- :meth:`DateOffset.__call__` is deprecated and will be removed in a future version, use ``offset + other`` instead (:issue:`34171`)
770+
- :meth:`DataFrame.tshift` and :meth:`Series.tshift` are deprecated and will be removed in a future version, use :meth:`DataFrame.shift` and :meth:`Series.shift` instead (:issue:`11631`)
770771
- Indexing an :class:`Index` object with a float key is deprecated, and will
771772
raise an ``IndexError`` in the future. You can manually convert to an integer key
772773
instead (:issue:`34191`).

pandas/core/generic.py

+100-59
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ class NDFrame(PandasObject, SelectionMixin, indexing.IndexingMixin):
182182
]
183183
_internal_names_set: Set[str] = set(_internal_names)
184184
_accessors: Set[str] = set()
185-
_deprecations: FrozenSet[str] = frozenset(["get_values"])
185+
_deprecations: FrozenSet[str] = frozenset(["get_values", "tshift"])
186186
_metadata: List[str] = []
187187
_is_copy = None
188188
_mgr: BlockManager
@@ -9162,7 +9162,9 @@ def shift(
91629162
When `freq` is not passed, shift the index without realigning the data.
91639163
If `freq` is passed (in this case, the index must be date or datetime,
91649164
or it will raise a `NotImplementedError`), the index will be
9165-
increased using the periods and the `freq`.
9165+
increased using the periods and the `freq`. `freq` can be inferred
9166+
when specified as "infer" as long as either freq or inferred_freq
9167+
attribute is set in the index.
91669168
91679169
Parameters
91689170
----------
@@ -9173,6 +9175,9 @@ def shift(
91739175
If `freq` is specified then the index values are shifted but the
91749176
data is not realigned. That is, use `freq` if you would like to
91759177
extend the index when shifting and preserve the original data.
9178+
If `freq` is specified as "infer" then it will be inferred from
9179+
the freq or inferred_freq attributes of the index. If neither of
9180+
those attributes exist, a ValueError is thrown
91769181
axis : {{0 or 'index', 1 or 'columns', None}}, default None
91779182
Shift direction.
91789183
fill_value : object, optional
@@ -9182,7 +9187,7 @@ def shift(
91829187
For datetime, timedelta, or period data, etc. :attr:`NaT` is used.
91839188
For extension dtypes, ``self.dtype.na_value`` is used.
91849189
9185-
.. versionchanged:: 0.24.0
9190+
.. versionchanged:: 1.1.0
91869191
91879192
Returns
91889193
-------
@@ -9199,46 +9204,99 @@ def shift(
91999204
92009205
Examples
92019206
--------
9202-
>>> df = pd.DataFrame({{'Col1': [10, 20, 15, 30, 45],
9203-
... 'Col2': [13, 23, 18, 33, 48],
9204-
... 'Col3': [17, 27, 22, 37, 52]}})
9207+
>>> df = pd.DataFrame({{"Col1": [10, 20, 15, 30, 45],
9208+
... "Col2": [13, 23, 18, 33, 48],
9209+
... "Col3": [17, 27, 22, 37, 52]}},
9210+
... index=pd.date_range("2020-01-01", "2020-01-05"))
9211+
>>> df
9212+
Col1 Col2 Col3
9213+
2020-01-01 10 13 17
9214+
2020-01-02 20 23 27
9215+
2020-01-03 15 18 22
9216+
2020-01-04 30 33 37
9217+
2020-01-05 45 48 52
92059218
92069219
>>> df.shift(periods=3)
9207-
Col1 Col2 Col3
9208-
0 NaN NaN NaN
9209-
1 NaN NaN NaN
9210-
2 NaN NaN NaN
9211-
3 10.0 13.0 17.0
9212-
4 20.0 23.0 27.0
9213-
9214-
>>> df.shift(periods=1, axis='columns')
9215-
Col1 Col2 Col3
9216-
0 NaN 10.0 13.0
9217-
1 NaN 20.0 23.0
9218-
2 NaN 15.0 18.0
9219-
3 NaN 30.0 33.0
9220-
4 NaN 45.0 48.0
9220+
Col1 Col2 Col3
9221+
2020-01-01 NaN NaN NaN
9222+
2020-01-02 NaN NaN NaN
9223+
2020-01-03 NaN NaN NaN
9224+
2020-01-04 10.0 13.0 17.0
9225+
2020-01-05 20.0 23.0 27.0
9226+
9227+
>>> df.shift(periods=1, axis="columns")
9228+
Col1 Col2 Col3
9229+
2020-01-01 NaN 10.0 13.0
9230+
2020-01-02 NaN 20.0 23.0
9231+
2020-01-03 NaN 15.0 18.0
9232+
2020-01-04 NaN 30.0 33.0
9233+
2020-01-05 NaN 45.0 48.0
92219234
92229235
>>> df.shift(periods=3, fill_value=0)
9223-
Col1 Col2 Col3
9224-
0 0 0 0
9225-
1 0 0 0
9226-
2 0 0 0
9227-
3 10 13 17
9228-
4 20 23 27
9236+
Col1 Col2 Col3
9237+
2020-01-01 0 0 0
9238+
2020-01-02 0 0 0
9239+
2020-01-03 0 0 0
9240+
2020-01-04 10 13 17
9241+
2020-01-05 20 23 27
9242+
9243+
>>> df.shift(periods=3, freq="D")
9244+
Col1 Col2 Col3
9245+
2020-01-04 10 13 17
9246+
2020-01-05 20 23 27
9247+
2020-01-06 15 18 22
9248+
2020-01-07 30 33 37
9249+
2020-01-08 45 48 52
9250+
9251+
>>> df.shift(periods=3, freq="infer")
9252+
Col1 Col2 Col3
9253+
2020-01-04 10 13 17
9254+
2020-01-05 20 23 27
9255+
2020-01-06 15 18 22
9256+
2020-01-07 30 33 37
9257+
2020-01-08 45 48 52
92299258
"""
92309259
if periods == 0:
92319260
return self.copy()
92329261

9233-
block_axis = self._get_block_manager_axis(axis)
92349262
if freq is None:
9263+
# when freq is None, data is shifted, index is not
9264+
block_axis = self._get_block_manager_axis(axis)
92359265
new_data = self._mgr.shift(
92369266
periods=periods, axis=block_axis, fill_value=fill_value
92379267
)
9268+
return self._constructor(new_data).__finalize__(self, method="shift")
9269+
9270+
# when freq is given, index is shifted, data is not
9271+
index = self._get_axis(axis)
9272+
9273+
if freq == "infer":
9274+
freq = getattr(index, "freq", None)
9275+
9276+
if freq is None:
9277+
freq = getattr(index, "inferred_freq", None)
9278+
9279+
if freq is None:
9280+
msg = "Freq was not set in the index hence cannot be inferred"
9281+
raise ValueError(msg)
9282+
9283+
elif isinstance(freq, str):
9284+
freq = to_offset(freq)
9285+
9286+
if isinstance(index, PeriodIndex):
9287+
orig_freq = to_offset(index.freq)
9288+
if freq != orig_freq:
9289+
assert orig_freq is not None # for mypy
9290+
raise ValueError(
9291+
f"Given freq {freq.rule_code} does not match "
9292+
f"PeriodIndex freq {orig_freq.rule_code}"
9293+
)
9294+
new_ax = index.shift(periods)
92389295
else:
9239-
return self.tshift(periods, freq)
9296+
new_ax = index.shift(periods, freq)
92409297

9241-
return self._constructor(new_data).__finalize__(self, method="shift")
9298+
result = self.set_axis(new_ax, axis)
9299+
return result.__finalize__(self, method="shift")
92429300

92439301
def slice_shift(self: FrameOrSeries, periods: int = 1, axis=0) -> FrameOrSeries:
92449302
"""
@@ -9283,6 +9341,9 @@ def tshift(
92839341
"""
92849342
Shift the time index, using the index's frequency if available.
92859343
9344+
.. deprecated:: 1.1.0
9345+
Use `shift` instead.
9346+
92869347
Parameters
92879348
----------
92889349
periods : int
@@ -9303,39 +9364,19 @@ def tshift(
93039364
attributes of the index. If neither of those attributes exist, a
93049365
ValueError is thrown
93059366
"""
9306-
index = self._get_axis(axis)
9307-
if freq is None:
9308-
freq = getattr(index, "freq", None)
9309-
9310-
if freq is None:
9311-
freq = getattr(index, "inferred_freq", None)
9367+
warnings.warn(
9368+
(
9369+
"tshift is deprecated and will be removed in a future version. "
9370+
"Please use shift instead."
9371+
),
9372+
FutureWarning,
9373+
stacklevel=2,
9374+
)
93129375

93139376
if freq is None:
9314-
msg = "Freq was not given and was not set in the index"
9315-
raise ValueError(msg)
9316-
9317-
if periods == 0:
9318-
return self
9319-
9320-
if isinstance(freq, str):
9321-
freq = to_offset(freq)
9322-
9323-
axis = self._get_axis_number(axis)
9324-
if isinstance(index, PeriodIndex):
9325-
orig_freq = to_offset(index.freq)
9326-
if freq != orig_freq:
9327-
assert orig_freq is not None # for mypy
9328-
raise ValueError(
9329-
f"Given freq {freq.rule_code} does not match "
9330-
f"PeriodIndex freq {orig_freq.rule_code}"
9331-
)
9332-
new_ax = index.shift(periods)
9333-
else:
9334-
new_ax = index.shift(periods, freq)
9377+
freq = "infer"
93359378

9336-
result = self.copy()
9337-
result.set_axis(new_ax, axis, inplace=True)
9338-
return result.__finalize__(self, method="tshift")
9379+
return self.shift(periods, freq, axis)
93399380

93409381
def truncate(
93419382
self: FrameOrSeries, before=None, after=None, axis=None, copy: bool_t = True

pandas/tests/frame/methods/test_shift.py

+57-2
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,10 @@ def test_shift_duplicate_columns(self):
145145
tm.assert_frame_equal(shifted[0], shifted[1])
146146
tm.assert_frame_equal(shifted[0], shifted[2])
147147

148+
@pytest.mark.filterwarnings("ignore:tshift is deprecated:FutureWarning")
148149
def test_tshift(self, datetime_frame):
150+
# TODO: remove this test when tshift deprecation is enforced
151+
149152
# PeriodIndex
150153
ps = tm.makePeriodFrame()
151154
shifted = ps.tshift(1)
@@ -159,7 +162,8 @@ def test_tshift(self, datetime_frame):
159162
shifted3 = ps.tshift(freq=offsets.BDay())
160163
tm.assert_frame_equal(shifted, shifted3)
161164

162-
with pytest.raises(ValueError, match="does not match"):
165+
msg = "Given freq M does not match PeriodIndex freq B"
166+
with pytest.raises(ValueError, match=msg):
163167
ps.tshift(freq="M")
164168

165169
# DatetimeIndex
@@ -186,10 +190,61 @@ def test_tshift(self, datetime_frame):
186190
tm.assert_frame_equal(unshifted, inferred_ts)
187191

188192
no_freq = datetime_frame.iloc[[0, 5, 7], :]
189-
msg = "Freq was not given and was not set in the index"
193+
msg = "Freq was not set in the index hence cannot be inferred"
190194
with pytest.raises(ValueError, match=msg):
191195
no_freq.tshift()
192196

197+
def test_tshift_deprecated(self, datetime_frame):
198+
# GH#11631
199+
with tm.assert_produces_warning(FutureWarning):
200+
datetime_frame.tshift()
201+
202+
def test_period_index_frame_shift_with_freq(self):
203+
ps = tm.makePeriodFrame()
204+
205+
shifted = ps.shift(1, freq="infer")
206+
unshifted = shifted.shift(-1, freq="infer")
207+
tm.assert_frame_equal(unshifted, ps)
208+
209+
shifted2 = ps.shift(freq="B")
210+
tm.assert_frame_equal(shifted, shifted2)
211+
212+
shifted3 = ps.shift(freq=offsets.BDay())
213+
tm.assert_frame_equal(shifted, shifted3)
214+
215+
def test_datetime_frame_shift_with_freq(self, datetime_frame):
216+
shifted = datetime_frame.shift(1, freq="infer")
217+
unshifted = shifted.shift(-1, freq="infer")
218+
tm.assert_frame_equal(datetime_frame, unshifted)
219+
220+
shifted2 = datetime_frame.shift(freq=datetime_frame.index.freq)
221+
tm.assert_frame_equal(shifted, shifted2)
222+
223+
inferred_ts = DataFrame(
224+
datetime_frame.values,
225+
Index(np.asarray(datetime_frame.index)),
226+
columns=datetime_frame.columns,
227+
)
228+
shifted = inferred_ts.shift(1, freq="infer")
229+
expected = datetime_frame.shift(1, freq="infer")
230+
expected.index = expected.index._with_freq(None)
231+
tm.assert_frame_equal(shifted, expected)
232+
233+
unshifted = shifted.shift(-1, freq="infer")
234+
tm.assert_frame_equal(unshifted, inferred_ts)
235+
236+
def test_period_index_frame_shift_with_freq_error(self):
237+
ps = tm.makePeriodFrame()
238+
msg = "Given freq M does not match PeriodIndex freq B"
239+
with pytest.raises(ValueError, match=msg):
240+
ps.shift(freq="M")
241+
242+
def test_datetime_frame_shift_with_freq_error(self, datetime_frame):
243+
no_freq = datetime_frame.iloc[[0, 5, 7], :]
244+
msg = "Freq was not set in the index hence cannot be inferred"
245+
with pytest.raises(ValueError, match=msg):
246+
no_freq.shift(freq="infer")
247+
193248
def test_shift_dt64values_int_fill_deprecated(self):
194249
# GH#31971
195250
ser = pd.Series([pd.Timestamp("2020-01-01"), pd.Timestamp("2020-01-02")])

pandas/tests/generic/test_finalize.py

+15-5
Original file line numberDiff line numberDiff line change
@@ -438,11 +438,21 @@
438438
(pd.DataFrame, frame_data, operator.methodcaller("mask", np.array([[True]]))),
439439
(pd.Series, ([1, 2],), operator.methodcaller("slice_shift")),
440440
(pd.DataFrame, frame_data, operator.methodcaller("slice_shift")),
441-
(pd.Series, (1, pd.date_range("2000", periods=4)), operator.methodcaller("tshift")),
442-
(
443-
pd.DataFrame,
444-
({"A": [1, 1, 1, 1]}, pd.date_range("2000", periods=4)),
445-
operator.methodcaller("tshift"),
441+
pytest.param(
442+
(
443+
pd.Series,
444+
(1, pd.date_range("2000", periods=4)),
445+
operator.methodcaller("tshift"),
446+
),
447+
marks=pytest.mark.filterwarnings("ignore::FutureWarning"),
448+
),
449+
pytest.param(
450+
(
451+
pd.DataFrame,
452+
({"A": [1, 1, 1, 1]}, pd.date_range("2000", periods=4)),
453+
operator.methodcaller("tshift"),
454+
),
455+
marks=pytest.mark.filterwarnings("ignore::FutureWarning"),
446456
),
447457
(pd.Series, ([1, 2],), operator.methodcaller("truncate", before=0)),
448458
(pd.DataFrame, frame_data, operator.methodcaller("truncate", before=0)),

pandas/tests/groupby/test_groupby.py

+1
Original file line numberDiff line numberDiff line change
@@ -1982,6 +1982,7 @@ def test_bool_aggs_dup_column_labels(bool_agg_func):
19821982
@pytest.mark.parametrize(
19831983
"idx", [pd.Index(["a", "a"]), pd.MultiIndex.from_tuples((("a", "a"), ("a", "a")))]
19841984
)
1985+
@pytest.mark.filterwarnings("ignore:tshift is deprecated:FutureWarning")
19851986
def test_dup_labels_output_shape(groupby_func, idx):
19861987
if groupby_func in {"size", "ngroup", "cumcount"}:
19871988
pytest.skip("Not applicable")

pandas/tests/groupby/test_groupby_subclass.py

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
tm.SubclassedSeries(np.arange(0, 10), name="A"),
1515
],
1616
)
17+
@pytest.mark.filterwarnings("ignore:tshift is deprecated:FutureWarning")
1718
def test_groupby_preserves_subclass(obj, groupby_func):
1819
# GH28330 -- preserve subclass through groupby operations
1920

0 commit comments

Comments
 (0)