From 072e1a97d8607cc565a7dddc24b80b8e2dc7a9c8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 5 Jan 2018 10:19:33 -0800 Subject: [PATCH 1/4] Fix TimedeltaIndex +/- offset array --- doc/source/whatsnew/v0.23.0.txt | 2 +- pandas/core/indexes/timedeltas.py | 47 ++++++++- .../indexes/timedeltas/test_arithmetic.py | 96 +++++++++++++++++-- 3 files changed, 132 insertions(+), 13 deletions(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index a62a737fbba31..389d167437909 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -308,7 +308,7 @@ Conversion - Bug in :class:`WeekOfMonth` and class:`Week` where addition and subtraction did not roll correctly (:issue:`18510`,:issue:`18672`,:issue:`18864`) - Bug in :meth:`DatetimeIndex.astype` when converting between timezone aware dtypes, and converting from timezone aware to naive (:issue:`18951`) - Bug in :class:`FY5253` where ``datetime`` addition and subtraction incremented incorrectly for dates on the year-end but not normalized to midnight (:issue:`18854`) -- Bug in :class:`DatetimeIndex` where adding or subtracting an array-like of ``DateOffset`` objects either raised (``np.array``, ``pd.Index``) or broadcast incorrectly (``pd.Series``) (:issue:`18849`) +- Bug in :class:`DatetimeIndex` and :class:`TimedeltaIndex` where adding or subtracting an array-like of ``DateOffset`` objects either raised (``np.array``, ``pd.Index``) or broadcast incorrectly (``pd.Series``) (:issue:`18849`) - Bug in :class:`Series` floor-division where operating on a scalar ``timedelta`` raises an exception (:issue:`18846`) - Bug in :class:`FY5253Quarter`, :class:`LastWeekOfMonth` where rollback and rollforward behavior was inconsistent with addition and subtraction behavior (:issue:`18854`) - Bug in :class:`Index` constructor with ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19032`) diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index d28a09225e8b8..984e2a26d8c95 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -1,6 +1,8 @@ """ implement the TimedeltaIndex """ from datetime import timedelta +import warnings + import numpy as np from pandas.core.dtypes.common import ( _TD_DTYPE, @@ -364,8 +366,8 @@ def _add_delta(self, delta): # update name when delta is index name = com._maybe_match_name(self, delta) else: - raise ValueError("cannot add the type {0} to a TimedeltaIndex" - .format(type(delta))) + raise TypeError("cannot add the type {0} to a TimedeltaIndex" + .format(type(delta))) result = TimedeltaIndex(new_values, freq='infer', name=name) return result @@ -411,6 +413,47 @@ def _sub_datelike(self, other): raise TypeError("cannot subtract a datelike from a TimedeltaIndex") return DatetimeIndex(result, name=self.name, copy=False) + def _add_offset_array(self, other): + # Array/Index of DateOffset objects + try: + # TimedeltaIndex can only operate with a subset of DateOffset + # subclasses. Incompatible classes will raise AttributeError, + # which we re-raise as TypeError + if isinstance(other, ABCSeries): + return NotImplemented + elif len(other) == 1: + return self + other[0] + else: + from pandas.errors import PerformanceWarning + warnings.warn("Adding/subtracting array of DateOffsets to " + "{} not vectorized".format(type(self)), + PerformanceWarning) + return self.astype('O') + np.array(other) + # TODO: This works for __add__ but loses dtype in __sub__ + except AttributeError: + raise TypeError("Cannot add non-tick DateOffset to TimedeltaIndex") + + def _sub_offset_array(self, other): + # Array/Index of DateOffset objects + try: + # TimedeltaIndex can only operate with a subset of DateOffset + # subclasses. Incompatible classes will raise AttributeError, + # which we re-raise as TypeError + if isinstance(other, ABCSeries): + return NotImplemented + elif len(other) == 1: + return self - other[0] + else: + from pandas.errors import PerformanceWarning + warnings.warn("Adding/subtracting array of DateOffsets to " + "{} not vectorized".format(type(self)), + PerformanceWarning) + res_values = self.astype('O').values - np.array(other) + return self.__class__(res_values, freq='infer') + except AttributeError: + raise TypeError("Cannot subtrack non-tick DateOffset from" + " TimedeltaIndex") + def _format_native_types(self, na_rep=u('NaT'), date_format=None, **kwargs): from pandas.io.formats.format import Timedelta64Formatter diff --git a/pandas/tests/indexes/timedeltas/test_arithmetic.py b/pandas/tests/indexes/timedeltas/test_arithmetic.py index 3ecfcaff63bc5..2014c6c33b11f 100644 --- a/pandas/tests/indexes/timedeltas/test_arithmetic.py +++ b/pandas/tests/indexes/timedeltas/test_arithmetic.py @@ -10,6 +10,7 @@ to_timedelta, timedelta_range, date_range, Series, Timestamp, Timedelta) +from pandas.errors import PerformanceWarning @pytest.fixture(params=[pd.offsets.Hour(2), timedelta(hours=2), @@ -28,23 +29,98 @@ def freq(request): class TestTimedeltaIndexArithmetic(object): _holder = TimedeltaIndex - @pytest.mark.xfail(reason='GH#18824 ufunc add cannot use operands...') - def test_tdi_with_offset_array(self): + @pytest.mark.parametrize('box', [np.array, pd.Index]) + def test_tdi_add_offset_array(self, box): # GH#18849 - tdi = pd.TimedeltaIndex(['1 days 00:00:00', '3 days 04:00:00']) - offs = np.array([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)]) - expected = pd.TimedeltaIndex(['1 days 01:00:00', '3 days 04:02:00']) + tdi = TimedeltaIndex(['1 days 00:00:00', '3 days 04:00:00']) + other = np.array([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)]) - res = tdi + offs + expected = TimedeltaIndex([tdi[n] + other[n] for n in range(len(tdi))], + name=tdi.name, freq='infer') + + with tm.assert_produces_warning(PerformanceWarning): + res = tdi + other tm.assert_index_equal(res, expected) - res2 = offs + tdi + with tm.assert_produces_warning(PerformanceWarning): + res2 = other + tdi tm.assert_index_equal(res2, expected) - anchored = np.array([pd.offsets.QuarterEnd(), - pd.offsets.Week(weekday=2)]) + anchored = box([pd.offsets.QuarterEnd(), + pd.offsets.Week(weekday=2)]) + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + tdi + anchored + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + anchored + tdi + + @pytest.mark.parametrize('box', [np.array, pd.Index]) + def test_tdi_sub_offset_array(self, box): + # GH#18824 + tdi = TimedeltaIndex(['1 days 00:00:00', '3 days 04:00:00']) + other = np.array([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)]) + + expected = TimedeltaIndex([tdi[n] - other[n] for n in range(len(tdi))], + name=tdi.name, freq='infer') + + with tm.assert_produces_warning(PerformanceWarning): + res = tdi - other + tm.assert_index_equal(res, expected) + + anchored = box([pd.offsets.MonthEnd(), pd.offsets.Day(n=2)]) + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + tdi - anchored + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + anchored - tdi + + @pytest.mark.parametrize('names', [(None, None, None), + ('foo', 'bar', None), + ('foo', 'foo', 'foo')]) + def test_tdi_with_offset_series(self, names): + # GH#18849 + tdi = TimedeltaIndex(['1 days 00:00:00', '3 days 04:00:00'], + name=names[0]) + other = Series([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)], + name=names[1]) + + expected_add = Series([tdi[n] + other[n] for n in range(len(tdi))], + name=names[2]) + + with tm.assert_produces_warning(PerformanceWarning): + res = tdi + other + tm.assert_series_equal(res, expected_add) + + with tm.assert_produces_warning(PerformanceWarning): + res2 = other + tdi + tm.assert_series_equal(res2, expected_add) + + expected_sub = Series([tdi[n] - other[n] for n in range(len(tdi))], + name=names[2]) + + with tm.assert_produces_warning(PerformanceWarning): + res3 = tdi - other + tm.assert_series_equal(res3, expected_sub) + + anchored = Series([pd.offsets.MonthEnd(), pd.offsets.Day(n=2)], + name=names[1]) + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + anchored + tdi + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + tdi + anchored + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + anchored + tdi + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + tdi - anchored with pytest.raises(TypeError): - tdi + anchored + with tm.assert_produces_warning(PerformanceWarning): + anchored - tdi # TODO: Split by ops, better name def test_numeric_compat(self): From e7ed242743983dc51a8a3eda2b0fdfb5fb2bd4f6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 5 Jan 2018 13:02:16 -0800 Subject: [PATCH 2/4] add missing box=Index test case, fix bug that had been missed --- pandas/core/indexes/datetimelike.py | 20 ++++---- .../indexes/timedeltas/test_arithmetic.py | 51 ++++++++----------- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index ee2fdd213dd9a..d26dbeb498b7a 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -675,20 +675,20 @@ def __add__(self, other): return NotImplemented elif is_timedelta64_dtype(other): return self._add_delta(other) + elif isinstance(other, (DateOffset, timedelta)): + return self._add_delta(other) + elif is_offsetlike(other): + # Array/Index of DateOffset objects + return self._add_offset_array(other) elif isinstance(self, TimedeltaIndex) and isinstance(other, Index): if hasattr(other, '_add_delta'): return other._add_delta(self) raise TypeError("cannot add TimedeltaIndex and {typ}" .format(typ=type(other))) - elif isinstance(other, (DateOffset, timedelta)): - return self._add_delta(other) elif is_integer(other): return self.shift(other) elif isinstance(other, (datetime, np.datetime64)): return self._add_datelike(other) - elif is_offsetlike(other): - # Array/Index of DateOffset objects - return self._add_offset_array(other) elif isinstance(other, Index): return self._add_datelike(other) else: # pragma: no cover @@ -708,6 +708,11 @@ def __sub__(self, other): return NotImplemented elif is_timedelta64_dtype(other): return self._add_delta(-other) + elif isinstance(other, (DateOffset, timedelta)): + return self._add_delta(-other) + elif is_offsetlike(other): + # Array/Index of DateOffset objects + return self._sub_offset_array(other) elif isinstance(self, TimedeltaIndex) and isinstance(other, Index): if not isinstance(other, TimedeltaIndex): raise TypeError("cannot subtract TimedeltaIndex and {typ}" @@ -715,17 +720,12 @@ def __sub__(self, other): return self._add_delta(-other) elif isinstance(other, DatetimeIndex): return self._sub_datelike(other) - elif isinstance(other, (DateOffset, timedelta)): - return self._add_delta(-other) elif is_integer(other): return self.shift(-other) elif isinstance(other, (datetime, np.datetime64)): return self._sub_datelike(other) elif isinstance(other, Period): return self._sub_period(other) - elif is_offsetlike(other): - # Array/Index of DateOffset objects - return self._sub_offset_array(other) elif isinstance(other, Index): raise TypeError("cannot subtract {typ1} and {typ2}" .format(typ1=type(self).__name__, diff --git a/pandas/tests/indexes/timedeltas/test_arithmetic.py b/pandas/tests/indexes/timedeltas/test_arithmetic.py index 2014c6c33b11f..0799c563e75ce 100644 --- a/pandas/tests/indexes/timedeltas/test_arithmetic.py +++ b/pandas/tests/indexes/timedeltas/test_arithmetic.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import operator + import pytest import numpy as np from datetime import timedelta @@ -26,6 +28,12 @@ def freq(request): return request.param +def _raises_and_warns(op, left, right): + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + op(left, right) + + class TestTimedeltaIndexArithmetic(object): _holder = TimedeltaIndex @@ -33,10 +41,10 @@ class TestTimedeltaIndexArithmetic(object): def test_tdi_add_offset_array(self, box): # GH#18849 tdi = TimedeltaIndex(['1 days 00:00:00', '3 days 04:00:00']) - other = np.array([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)]) + other = box([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)]) expected = TimedeltaIndex([tdi[n] + other[n] for n in range(len(tdi))], - name=tdi.name, freq='infer') + freq='infer') with tm.assert_produces_warning(PerformanceWarning): res = tdi + other @@ -48,33 +56,25 @@ def test_tdi_add_offset_array(self, box): anchored = box([pd.offsets.QuarterEnd(), pd.offsets.Week(weekday=2)]) - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - tdi + anchored - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - anchored + tdi + _raises_and_warns(operator.add, tdi, anchored) + _raises_and_warns(operator.add, anchored, tdi) @pytest.mark.parametrize('box', [np.array, pd.Index]) def test_tdi_sub_offset_array(self, box): # GH#18824 tdi = TimedeltaIndex(['1 days 00:00:00', '3 days 04:00:00']) - other = np.array([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)]) + other = box([pd.offsets.Hour(n=1), pd.offsets.Minute(n=-2)]) expected = TimedeltaIndex([tdi[n] - other[n] for n in range(len(tdi))], - name=tdi.name, freq='infer') + freq='infer') with tm.assert_produces_warning(PerformanceWarning): res = tdi - other tm.assert_index_equal(res, expected) anchored = box([pd.offsets.MonthEnd(), pd.offsets.Day(n=2)]) - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - tdi - anchored - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - anchored - tdi + _raises_and_warns(operator.sub, tdi, anchored) + _raises_and_warns(operator.sub, anchored, tdi) @pytest.mark.parametrize('names', [(None, None, None), ('foo', 'bar', None), @@ -106,21 +106,10 @@ def test_tdi_with_offset_series(self, names): anchored = Series([pd.offsets.MonthEnd(), pd.offsets.Day(n=2)], name=names[1]) - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - anchored + tdi - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - tdi + anchored - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - anchored + tdi - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - tdi - anchored - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - anchored - tdi + _raises_and_warns(operator.add, anchored, tdi) + _raises_and_warns(operator.add, tdi, anchored) + _raises_and_warns(operator.sub, anchored, tdi) + _raises_and_warns(operator.sub, tdi, anchored) # TODO: Split by ops, better name def test_numeric_compat(self): From 49b59201d54e8e9487a54369e7b4db9bcf072215 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 5 Jan 2018 18:13:03 -0800 Subject: [PATCH 3/4] revert raises_and_warns --- .../indexes/timedeltas/test_arithmetic.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/pandas/tests/indexes/timedeltas/test_arithmetic.py b/pandas/tests/indexes/timedeltas/test_arithmetic.py index 0799c563e75ce..ff966a26555f2 100644 --- a/pandas/tests/indexes/timedeltas/test_arithmetic.py +++ b/pandas/tests/indexes/timedeltas/test_arithmetic.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -import operator - import pytest import numpy as np from datetime import timedelta @@ -28,12 +26,6 @@ def freq(request): return request.param -def _raises_and_warns(op, left, right): - with pytest.raises(TypeError): - with tm.assert_produces_warning(PerformanceWarning): - op(left, right) - - class TestTimedeltaIndexArithmetic(object): _holder = TimedeltaIndex @@ -56,8 +48,12 @@ def test_tdi_add_offset_array(self, box): anchored = box([pd.offsets.QuarterEnd(), pd.offsets.Week(weekday=2)]) - _raises_and_warns(operator.add, tdi, anchored) - _raises_and_warns(operator.add, anchored, tdi) + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + tdi + anchored + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + anchored + tdi @pytest.mark.parametrize('box', [np.array, pd.Index]) def test_tdi_sub_offset_array(self, box): @@ -73,8 +69,12 @@ def test_tdi_sub_offset_array(self, box): tm.assert_index_equal(res, expected) anchored = box([pd.offsets.MonthEnd(), pd.offsets.Day(n=2)]) - _raises_and_warns(operator.sub, tdi, anchored) - _raises_and_warns(operator.sub, anchored, tdi) + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + tdi - anchored + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + anchored - tdi @pytest.mark.parametrize('names', [(None, None, None), ('foo', 'bar', None), @@ -106,10 +106,18 @@ def test_tdi_with_offset_series(self, names): anchored = Series([pd.offsets.MonthEnd(), pd.offsets.Day(n=2)], name=names[1]) - _raises_and_warns(operator.add, anchored, tdi) - _raises_and_warns(operator.add, tdi, anchored) - _raises_and_warns(operator.sub, anchored, tdi) - _raises_and_warns(operator.sub, tdi, anchored) + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + tdi + anchored + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + anchored + tdi + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + tdi - anchored + with pytest.raises(TypeError): + with tm.assert_produces_warning(PerformanceWarning): + anchored - tdi # TODO: Split by ops, better name def test_numeric_compat(self): From b9e9fff3e1aabc9d02b81bc856a695b59b66411f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 6 Jan 2018 09:48:29 -0800 Subject: [PATCH 4/4] suggested comment --- pandas/tests/indexes/timedeltas/test_arithmetic.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pandas/tests/indexes/timedeltas/test_arithmetic.py b/pandas/tests/indexes/timedeltas/test_arithmetic.py index ff966a26555f2..2581a8fad078a 100644 --- a/pandas/tests/indexes/timedeltas/test_arithmetic.py +++ b/pandas/tests/indexes/timedeltas/test_arithmetic.py @@ -48,6 +48,9 @@ def test_tdi_add_offset_array(self, box): anchored = box([pd.offsets.QuarterEnd(), pd.offsets.Week(weekday=2)]) + + # addition/subtraction ops with anchored offsets should issue + # a PerformanceWarning and _then_ raise a TypeError. with pytest.raises(TypeError): with tm.assert_produces_warning(PerformanceWarning): tdi + anchored @@ -69,6 +72,9 @@ def test_tdi_sub_offset_array(self, box): tm.assert_index_equal(res, expected) anchored = box([pd.offsets.MonthEnd(), pd.offsets.Day(n=2)]) + + # addition/subtraction ops with anchored offsets should issue + # a PerformanceWarning and _then_ raise a TypeError. with pytest.raises(TypeError): with tm.assert_produces_warning(PerformanceWarning): tdi - anchored @@ -106,6 +112,9 @@ def test_tdi_with_offset_series(self, names): anchored = Series([pd.offsets.MonthEnd(), pd.offsets.Day(n=2)], name=names[1]) + + # addition/subtraction ops with anchored offsets should issue + # a PerformanceWarning and _then_ raise a TypeError. with pytest.raises(TypeError): with tm.assert_produces_warning(PerformanceWarning): tdi + anchored