From c4d4d108a2528498f933bf7768dff868b9e59cab Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 1 Aug 2018 14:20:33 -0700 Subject: [PATCH 1/9] API: Make Tick respect calendar time --- pandas/_libs/tslibs/timestamps.pyx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index be988e7247e59..e9066beccfedd 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -268,13 +268,17 @@ cdef class _Timestamp(datetime): "without freq.") return Timestamp((self.freq * other).apply(self), freq=self.freq) - elif PyDelta_Check(other) or hasattr(other, 'delta'): - # delta --> offsets.Tick + elif PyDelta_Check(other): nanos = delta_to_nanoseconds(other) result = Timestamp(self.value + nanos, tz=self.tzinfo, freq=self.freq) + return result + + elif hasattr(other, 'delta'): + # delta --> offsets.Tick + result = self.tz_localize(None) + other.delta + result = result.tz_localize(self.tz) if getattr(other, 'normalize', False): - # DateOffset result = result.normalize() return result From eae612f50abd1b2ba34403d3d45f3a3c1753c634 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 1 Aug 2018 21:55:11 -0700 Subject: [PATCH 2/9] Fix offset test --- pandas/tests/tseries/offsets/test_offsets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 57b9a281ac0eb..9bea427361dda 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -3035,7 +3035,7 @@ def test_springforward_singular(self): QuarterEnd: ['11/2/2012', '12/31/2012'], BQuarterBegin: ['11/2/2012', '12/3/2012'], BQuarterEnd: ['11/2/2012', '12/31/2012'], - Day: ['11/4/2012', '11/4/2012 23:00']}.items() + Day: ['11/4/2012', '11/5/2012']}.items() @pytest.mark.parametrize('tup', offset_classes) def test_all_offset_classes(self, tup): From 6a2eaff76d502ab96113493db9cf79766481897b Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 1 Aug 2018 22:37:55 -0700 Subject: [PATCH 3/9] Fix dst issue --- pandas/_libs/tslibs/timestamps.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index e9066beccfedd..1d60b132673ee 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -276,8 +276,9 @@ cdef class _Timestamp(datetime): elif hasattr(other, 'delta'): # delta --> offsets.Tick + dst = bool(self.dst()) result = self.tz_localize(None) + other.delta - result = result.tz_localize(self.tz) + result = result.tz_localize(self.tz, ambiguous=dst) if getattr(other, 'normalize', False): result = result.normalize() return result From 3fc886a2ce7fb40927dc95dfd097815231a2798a Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 1 Aug 2018 23:46:17 -0700 Subject: [PATCH 4/9] Add tests --- pandas/tests/tseries/offsets/test_offsets.py | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 9bea427361dda..17deb527a6668 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -3162,3 +3162,24 @@ def test_last_week_of_month_on_offset(): slow = (ts + offset) - offset == ts fast = offset.onOffset(ts) assert fast == slow + + +@pytest.mark.parametrize('offset, factor', [ + ['Day', 1], + ['Hour', 24], + ['Minute', 24 * 60], + ['Second', 24 * 60 * 60], + ['Milli', 8.64e7], + ['Micro', 8.64e10], + ['Nano', 8.64e13] +]) +def test_tick_arithmetic(offset, factor): + # GH + # Tick classes (e.g. Day) should now respect calendar arithmetic + # The most evident examples are DST crossing + ts = Timestamp('2016-10-30 00:00:00+0300', tz='Europe/Helsinki') + expected = Timestamp('2016-10-31 00:00:00+0200', tz='Europe/Helsinki') + + tick = factor * getattr(offsets, offset)(1) + result = ts + tick + assert result == expected From 4d2a2b6549db40ba5a80cdd7766128b7c6257eda Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 1 Aug 2018 23:51:07 -0700 Subject: [PATCH 5/9] clarify test --- pandas/tests/tseries/offsets/test_offsets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index 17deb527a6668..bf31aa850b7d3 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -3174,9 +3174,9 @@ def test_last_week_of_month_on_offset(): ['Nano', 8.64e13] ]) def test_tick_arithmetic(offset, factor): - # GH + # GH 20633 # Tick classes (e.g. Day) should now respect calendar arithmetic - # The most evident examples are DST crossing + # Test that calendar day is respected with a "fall back" DST transition ts = Timestamp('2016-10-30 00:00:00+0300', tz='Europe/Helsinki') expected = Timestamp('2016-10-31 00:00:00+0200', tz='Europe/Helsinki') From 469f5d5cdad770b24f4e0ae0380e37bfa6b1c9e8 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 2 Aug 2018 16:26:21 -0700 Subject: [PATCH 6/9] Test subtraction too --- pandas/tests/tseries/offsets/test_offsets.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index bf31aa850b7d3..c2e4b31138165 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -3176,10 +3176,13 @@ def test_last_week_of_month_on_offset(): def test_tick_arithmetic(offset, factor): # GH 20633 # Tick classes (e.g. Day) should now respect calendar arithmetic - # Test that calendar day is respected with a "fall back" DST transition + # Test that calendar day is respected by roundtripping across DST + tick = factor * getattr(offsets, offset)(1) ts = Timestamp('2016-10-30 00:00:00+0300', tz='Europe/Helsinki') expected = Timestamp('2016-10-31 00:00:00+0200', tz='Europe/Helsinki') - tick = factor * getattr(offsets, offset)(1) result = ts + tick assert result == expected + + result = result - tick + assert result == ts From 84eb6794509278ca8f1da2260af4a03ce889eef2 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 2 Aug 2018 21:56:24 -0700 Subject: [PATCH 7/9] Change DatetimeIndex arithmetic with Ticks --- pandas/core/arrays/datetimes.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index ee0677f760705..4d88c74c834f6 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -466,22 +466,19 @@ def _sub_datelike_dti(self, other): return new_values.view('timedelta64[ns]') def _add_offset(self, offset): - assert not isinstance(offset, Tick) try: if self.tz is not None: values = self.tz_localize(None) else: values = self result = offset.apply_index(values) - if self.tz is not None: - result = result.tz_localize(self.tz) except NotImplementedError: warnings.warn("Non-vectorized DateOffset being applied to Series " "or DatetimeIndex", PerformanceWarning) result = self.astype('O') + offset - return type(self)(result, freq='infer') + return type(self)(result, freq='infer', tz=self.tz) def _sub_datelike(self, other): # subtract a datetime from myself, yielding a ndarray[timedelta64[ns]] @@ -536,8 +533,12 @@ def _add_delta(self, delta): method (__add__ or __sub__) """ from pandas.core.arrays.timedeltas import TimedeltaArrayMixin - - if isinstance(delta, (Tick, timedelta, np.timedelta64)): + if isinstance(delta, Tick): + # GH 20633: Ticks behave like offsets now, but cannot change + # this directly in the mixin because it affects Periods and + # Timedeltas + return self._add_offset(delta) + elif isinstance(delta, (timedelta, np.timedelta64)): new_values = self._add_delta_td(delta) elif is_timedelta64_dtype(delta): if not isinstance(delta, TimedeltaArrayMixin): From 8676f6365dd788a97eb0a3390553ad89bece7fee Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 3 Aug 2018 15:53:32 -0700 Subject: [PATCH 8/9] Add DatetimeIndex tests and impliment apply_index --- .../indexes/datetimes/test_arithmetic.py | 25 +++++++++++++++++++ pandas/tests/tseries/offsets/test_offsets.py | 2 +- pandas/tseries/offsets.py | 18 +++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index f54cb32b0a036..d2c6d8d05f2ce 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -1106,6 +1106,31 @@ def test_dti_add_offset_tzaware(self, tz_aware_fixture): offset = dates + timedelta(hours=5) tm.assert_index_equal(offset, expected) + @pytest.mark.parametrize('offset, factor', [ + ['Day', 1], + ['Hour', 24], + ['Minute', 24 * 60], + ['Second', 24 * 60 * 60], + ['Milli', 8.64e7], + ['Micro', 8.64e10], + ['Nano', 8.64e13] + ]) + def test_dti_tick_arithmetic_dst_roundtrip(self, offset, factor): + # GH 20633 + # Tick classes (e.g. Day) should now respect calendar arithmetic + # Test that calendar day is respected by roundtripping across DST + tick = factor * getattr(pd.offsets, offset)(1) + ts = Timestamp('2016-10-30 00:00:00+0300', tz='Europe/Helsinki') + dti = DatetimeIndex([ts]) + expected = Timestamp('2016-10-31 00:00:00+0200', tz='Europe/Helsinki') + expected = DatetimeIndex([expected]) + + result = dti + tick + tm.assert_index_equal(result, expected) + + result = result - tick + tm.assert_index_equal(result, dti) + @pytest.mark.parametrize('klass,assert_func', [ (Series, tm.assert_series_equal), diff --git a/pandas/tests/tseries/offsets/test_offsets.py b/pandas/tests/tseries/offsets/test_offsets.py index c2e4b31138165..1314073aaccd0 100644 --- a/pandas/tests/tseries/offsets/test_offsets.py +++ b/pandas/tests/tseries/offsets/test_offsets.py @@ -3173,7 +3173,7 @@ def test_last_week_of_month_on_offset(): ['Micro', 8.64e10], ['Nano', 8.64e13] ]) -def test_tick_arithmetic(offset, factor): +def test_tick_dst_arithmetic(offset, factor): # GH 20633 # Tick classes (e.g. Day) should now respect calendar arithmetic # Test that calendar day is respected by roundtripping across DST diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 9d41401a7eefc..13bc0a5bb808f 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -2229,6 +2229,24 @@ def apply(self, other): raise ApplyTypeError('Unhandled type: {type_str}' .format(type_str=type(other).__name__)) + def apply_index(self, idx): + """ + Vectorized apply (addition) of Tick to DatetimeIndex + + Parameters + ---------- + idx : DatetimeIndex + + Returns + ------- + DatetimeIndex + """ + # TODO: Add a vectorized DatetimeIndex.dst() method + ambiguous = np.array([bool(ts.dst()) if ts is not tslibs.NaT else False + for ts in idx]) + result = idx.tz_localize(None) + self.delta + return result.tz_localize(idx.tz, ambiguous=ambiguous) + def isAnchored(self): return False From c16980c6e3949fda429169e72309a261dd681fca Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 3 Aug 2018 16:06:29 -0700 Subject: [PATCH 9/9] Add series test --- pandas/tests/series/test_arithmetic.py | 25 +++++++++++++++++++++++++ pandas/tseries/offsets.py | 1 - 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pandas/tests/series/test_arithmetic.py b/pandas/tests/series/test_arithmetic.py index c091df63fcfc7..3a9cdd4bedcdc 100644 --- a/pandas/tests/series/test_arithmetic.py +++ b/pandas/tests/series/test_arithmetic.py @@ -505,6 +505,31 @@ def test_ops_series_timedelta(self): result = pd.tseries.offsets.Day() + ser tm.assert_series_equal(result, expected) + @pytest.mark.parametrize('offset, factor', [ + ['Day', 1], + ['Hour', 24], + ['Minute', 24 * 60], + ['Second', 24 * 60 * 60], + ['Milli', 8.64e7], + ['Micro', 8.64e10], + ['Nano', 8.64e13] + ]) + def test_series_tick_arithmetic_dst_roundtrip(self, offset, factor): + # GH 20633 + # Tick classes (e.g. Day) should now respect calendar arithmetic + # Test that calendar day is respected by roundtripping across DST + tick = factor * getattr(pd.offsets, offset)(1) + ts = Timestamp('2016-10-30 00:00:00+0300', tz='Europe/Helsinki') + s = Series([ts]) + expected = Timestamp('2016-10-31 00:00:00+0200', tz='Europe/Helsinki') + expected = Series([expected]) + + result = s + tick + tm.assert_series_equal(result, expected) + + result = result - tick + tm.assert_series_equal(result, s) + def test_ops_series_period(self): # GH 13043 ser = pd.Series([pd.Period('2015-01-01', freq='D'), diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py index 13bc0a5bb808f..5616d5b9b3787 100644 --- a/pandas/tseries/offsets.py +++ b/pandas/tseries/offsets.py @@ -2204,7 +2204,6 @@ def delta(self): def nanos(self): return delta_to_nanoseconds(self.delta) - # TODO: Should Tick have its own apply_index? def apply(self, other): # Timestamp can handle tz and nano sec, thus no need to use apply_wraps if isinstance(other, Timestamp):