From e9dd35bd7979214ee9377a0b880faf7e235c85ab Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 10:22:28 -0700 Subject: [PATCH 01/13] masking and overflow checks for datetimeindex and timedeltaindex ops WIP filling out a test matrix of arithmetic ops closes #17991 --- pandas/core/indexes/datetimes.py | 4 +- pandas/core/indexes/timedeltas.py | 3 +- pandas/tests/indexes/datetimelike.py | 182 ++++++++++++++++++++++++++- 3 files changed, 186 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 18be6c61abdf7..977fb45d5d778 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -29,6 +29,7 @@ import pandas.core.dtypes.concat as _concat from pandas.errors import PerformanceWarning from pandas.core.common import _values_from_object, _maybe_box +from pandas.core.algorithms import checked_add_with_arr from pandas.core.indexes.base import Index, _index_shared_docs from pandas.core.indexes.numeric import Int64Index, Float64Index @@ -777,7 +778,8 @@ def _sub_datelike(self, other): "timezones or no timezones") else: i8 = self.asi8 - result = i8 - other.value + result = checked_add_with_arr(i8, -other.value, + arr_mask=self._isnan) result = self._maybe_mask_results(result, fill_value=libts.iNaT) else: diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 6e08c32f30dcd..546d54eaa0e51 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -361,7 +361,8 @@ def _add_datelike(self, other): else: other = Timestamp(other) i8 = self.asi8 - result = checked_add_with_arr(i8, other.value) + result = checked_add_with_arr(i8, other.value, + arr_mask=self._isnan) result = self._maybe_mask_results(result, fill_value=iNaT) return DatetimeIndex(result, name=self.name, copy=False) diff --git a/pandas/tests/indexes/datetimelike.py b/pandas/tests/indexes/datetimelike.py index 12b509d4aef3f..1f68bb6c8a954 100644 --- a/pandas/tests/indexes/datetimelike.py +++ b/pandas/tests/indexes/datetimelike.py @@ -1,8 +1,11 @@ """ generic datetimelike tests """ +import pytest + +import pandas as pd from .common import Base import pandas.util.testing as tm - +from pandas import Timestamp, Timedelta, NaT class DatetimeLike(Base): @@ -38,3 +41,180 @@ def test_view(self, indices): i_view = i.view(self._holder) result = self._holder(i) tm.assert_index_equal(result, i_view) + + +class TestDatetimeLikeIndexArithmetic(object): + # GH17991 checking for overflows and NaT masking on arithmetic ops + + # TODO: Fill out the matrix of allowed arithmetic operations: + # - __rsub__, __radd__ + # - ops with scalars boxed in Index/Series/DataFrame + # - ops with scalars: + # NaT, Timestamp.min/max, Timedelta.min/max + # datetime, timedelta, date(?), + # relativedelta, + # np.datetime64, np.timedelta64, + # DateOffset, + # Period + # - timezone-aware variants + # - PeriodIndex + # - consistency with .map(...) ? + + def test_timedeltaindex_add_timestamp_nat_masking(self): + tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) + + # tsneg.value < 0, tspos.value > 0 + tsneg = Timestamp('1950-01-01') + tspos = Timestamp('1980-01-01') + + res1 = tdinat + tsneg + assert res1[1] is NaT + res2 = tdinat + tspos + assert res2[1] is NaT + + def test_timedeltaindex_sub_timestamp_nat_masking(self): + tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) + + # tsneg.value < 0, tspos.value > 0 + tsneg = Timestamp('1950-01-01') + tspos = Timestamp('1980-01-01') + + res1 = tdinat - tsneg + assert res1[1] is NaT + res2 = tdinat - tspos + assert res2[1] is NaT + + def test_timedeltaindex_add_timestamp_overflow(self): + tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) + tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) + + # tsneg.value < 0, tspos.value > 0 + tsneg = Timestamp('1950-01-01') + tspos = Timestamp('1980-01-01') + + res1 = tdimax + tsneg + res2 = tdimin + tspos + + with pytest.raises(OverflowError): + tdimax + tspos + + with pytest.raises(OverflowError): + tdimin + tsneg + + def test_timedeltaindex_add_timedelta_overflow(self): + tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) + tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) + + # tdpos.value > 0, tdneg.value < 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + with pytest.raises(OverflowError): + res1 = tdimax + tdpos + + res2 = tdimax + tdneg + + res3 = tdimin + tdpos + with pytest.raises(OverflowError): + res4 = tdimin + tdneg + + def test_timedeltaindex_sub_timedelta_overflow(self): + tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) + tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) + + # tdpos.value > 0, tdneg.value < 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + res1 = tdimax - tdpos + with pytest.raises(OverflowError): + res2 = tdimax - tdneg + with pytest.raises(OverflowError): + res3 = tdimin - tdpos + res4 = tdimin - tdneg + + def test_datetimeindex_add_nat_masking(self): + # Checking for NaTs and checking that we don't get an OverflowError + dtinat = pd.to_datetime(['now', 'NaT']) + + # tdpos.value > 0, tdneg.value < 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + res1 = dtinat + tdpos + assert res1[1] is NaT + res2 = dtinat + tdneg + assert res2[1] is NaT + + def test_datetimeindex_sub_nat_masking(self): + # Checking for NaTs and checking that we don't get an OverflowError + dtinat = pd.to_datetime(['now', 'NaT']) + + # tdpos.value > 0, tdneg.value < 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + res1 = dtinat - tdpos + assert res1[1] is NaT + res2 = dtinat - tdneg + assert res2[1] is NaT + + def test_datetimeindex_add_timedelta_overflow(self): + dtimax = pd.to_datetime(['now', Timestamp.max]) + dtimin = pd.to_datetime(['now', Timestamp.min]) + + # tdpos.value < 0, tdneg.value > 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + with pytest.raises(OverflowError): + res1 = dtimax + tdpos + + res2 = dtimax + tdneg + assert res2[1].value == Timestamp.max.value + tdneg.value + + res3 = dtimin + tdpos + assert res3[1].value == Timestamp.min.value + tdpos.value + + with pytest.raises(OverflowError): + res4 = dtimin + tdneg + + def test_datetimeindex_sub_timedelta_overflow(self): + dtimax = pd.to_datetime(['now', Timestamp.max]) + dtimin = pd.to_datetime(['now', Timestamp.min]) + + # tdpos.value < 0, tdneg.value > 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + res1 = dtimax - tdpos + assert res1[1].value == Timestamp.max.value - tdpos.value + + with pytest.raises(OverflowError): + res2 = dtimax - tdneg + + with pytest.raises(OverflowError): + res3 = dtimin - tdpos + + res4 = dtimin - tdneg + assert res4[1].value == Timestamp.min.value - tdneg.value + + def test_datetimeindex_sub_timestamp_overflow(self): + dtimax = pd.to_datetime(['now', Timestamp.max]) + dtimin = pd.to_datetime(['now', Timestamp.min]) + + # tsneg.value < 0, tspos.value > 0 + tsneg = Timestamp('1950-01-01') + tspos = Timestamp('1980-01-01') + + with pytest.raises(OverflowError): + res1 = dtimax - tsneg + + res2 = dtimax - tspos + assert res2[1].value == Timestamp.max.value - tspos.value + + res3 = dtimin - tsneg + assert res3[1].value == Timestamp.min.value - tsneg.value + + with pytest.raises(OverflowError): + res4 = dtimin - tspos From 36aea6979f1e0b1ec0f1083b652d9f0fac5ab67a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 10:26:47 -0700 Subject: [PATCH 02/13] flake8 fixup --- pandas/tests/indexes/datetimelike.py | 32 ++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/pandas/tests/indexes/datetimelike.py b/pandas/tests/indexes/datetimelike.py index 1f68bb6c8a954..e43ebce30904f 100644 --- a/pandas/tests/indexes/datetimelike.py +++ b/pandas/tests/indexes/datetimelike.py @@ -7,6 +7,7 @@ import pandas.util.testing as tm from pandas import Timestamp, Timedelta, NaT + class DatetimeLike(Base): def test_shift_identity(self): @@ -93,7 +94,9 @@ def test_timedeltaindex_add_timestamp_overflow(self): tspos = Timestamp('1980-01-01') res1 = tdimax + tsneg + assert res1[1].value == Timedelta.max.value + tsneg.value res2 = tdimin + tspos + assert res2[1].value == Timedelta.min.value + tspos.value with pytest.raises(OverflowError): tdimax + tspos @@ -110,13 +113,15 @@ def test_timedeltaindex_add_timedelta_overflow(self): tdneg = Timedelta('-1h') with pytest.raises(OverflowError): - res1 = tdimax + tdpos + tdimax + tdpos res2 = tdimax + tdneg - + assert res2[1].value == Timedelta.max.value + tdneg.value res3 = tdimin + tdpos + assert res3[1].value == Timedelta.min.value + tdpos.value + with pytest.raises(OverflowError): - res4 = tdimin + tdneg + tdimin + tdneg def test_timedeltaindex_sub_timedelta_overflow(self): tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) @@ -127,11 +132,16 @@ def test_timedeltaindex_sub_timedelta_overflow(self): tdneg = Timedelta('-1h') res1 = tdimax - tdpos + assert res1[1].value == Timedelta.max.value - tdpos.value + with pytest.raises(OverflowError): - res2 = tdimax - tdneg + tdimax - tdneg + with pytest.raises(OverflowError): - res3 = tdimin - tdpos + tdimin - tdpos + res4 = tdimin - tdneg + assert res4[1].value == Timedelta.min.value - tdneg.value def test_datetimeindex_add_nat_masking(self): # Checking for NaTs and checking that we don't get an OverflowError @@ -168,7 +178,7 @@ def test_datetimeindex_add_timedelta_overflow(self): tdneg = Timedelta('-1h') with pytest.raises(OverflowError): - res1 = dtimax + tdpos + dtimax + tdpos res2 = dtimax + tdneg assert res2[1].value == Timestamp.max.value + tdneg.value @@ -177,7 +187,7 @@ def test_datetimeindex_add_timedelta_overflow(self): assert res3[1].value == Timestamp.min.value + tdpos.value with pytest.raises(OverflowError): - res4 = dtimin + tdneg + dtimin + tdneg def test_datetimeindex_sub_timedelta_overflow(self): dtimax = pd.to_datetime(['now', Timestamp.max]) @@ -191,10 +201,10 @@ def test_datetimeindex_sub_timedelta_overflow(self): assert res1[1].value == Timestamp.max.value - tdpos.value with pytest.raises(OverflowError): - res2 = dtimax - tdneg + dtimax - tdneg with pytest.raises(OverflowError): - res3 = dtimin - tdpos + dtimin - tdpos res4 = dtimin - tdneg assert res4[1].value == Timestamp.min.value - tdneg.value @@ -208,7 +218,7 @@ def test_datetimeindex_sub_timestamp_overflow(self): tspos = Timestamp('1980-01-01') with pytest.raises(OverflowError): - res1 = dtimax - tsneg + dtimax - tsneg res2 = dtimax - tspos assert res2[1].value == Timestamp.max.value - tspos.value @@ -217,4 +227,4 @@ def test_datetimeindex_sub_timestamp_overflow(self): assert res3[1].value == Timestamp.min.value - tsneg.value with pytest.raises(OverflowError): - res4 = dtimin - tspos + dtimin - tspos From dd1eddda84bed30fd38a36fd53a24e579e7eed7e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 10:31:29 -0700 Subject: [PATCH 03/13] add todo notes --- pandas/tests/indexes/datetimelike.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimelike.py b/pandas/tests/indexes/datetimelike.py index e43ebce30904f..b6b2c299d19bd 100644 --- a/pandas/tests/indexes/datetimelike.py +++ b/pandas/tests/indexes/datetimelike.py @@ -49,7 +49,7 @@ class TestDatetimeLikeIndexArithmetic(object): # TODO: Fill out the matrix of allowed arithmetic operations: # - __rsub__, __radd__ - # - ops with scalars boxed in Index/Series/DataFrame + # - ops with scalars boxed in Index/Series/DataFrame/np.array # - ops with scalars: # NaT, Timestamp.min/max, Timedelta.min/max # datetime, timedelta, date(?), @@ -58,6 +58,7 @@ class TestDatetimeLikeIndexArithmetic(object): # DateOffset, # Period # - timezone-aware variants + # - object-dtype, categorical dtype # - PeriodIndex # - consistency with .map(...) ? From 92afc802800daca5763b5c9f06956e3f4e445903 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 12:32:29 -0700 Subject: [PATCH 04/13] Move test somewhere it will actually get run --- pandas/tests/indexes/datetimelike.py | 191 ----------------- .../indexes/test_datetimelike_arithmetic.py | 195 ++++++++++++++++++ 2 files changed, 195 insertions(+), 191 deletions(-) create mode 100644 pandas/tests/indexes/test_datetimelike_arithmetic.py diff --git a/pandas/tests/indexes/datetimelike.py b/pandas/tests/indexes/datetimelike.py index b6b2c299d19bd..12b509d4aef3f 100644 --- a/pandas/tests/indexes/datetimelike.py +++ b/pandas/tests/indexes/datetimelike.py @@ -1,11 +1,7 @@ """ generic datetimelike tests """ -import pytest - -import pandas as pd from .common import Base import pandas.util.testing as tm -from pandas import Timestamp, Timedelta, NaT class DatetimeLike(Base): @@ -42,190 +38,3 @@ def test_view(self, indices): i_view = i.view(self._holder) result = self._holder(i) tm.assert_index_equal(result, i_view) - - -class TestDatetimeLikeIndexArithmetic(object): - # GH17991 checking for overflows and NaT masking on arithmetic ops - - # TODO: Fill out the matrix of allowed arithmetic operations: - # - __rsub__, __radd__ - # - ops with scalars boxed in Index/Series/DataFrame/np.array - # - ops with scalars: - # NaT, Timestamp.min/max, Timedelta.min/max - # datetime, timedelta, date(?), - # relativedelta, - # np.datetime64, np.timedelta64, - # DateOffset, - # Period - # - timezone-aware variants - # - object-dtype, categorical dtype - # - PeriodIndex - # - consistency with .map(...) ? - - def test_timedeltaindex_add_timestamp_nat_masking(self): - tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) - - # tsneg.value < 0, tspos.value > 0 - tsneg = Timestamp('1950-01-01') - tspos = Timestamp('1980-01-01') - - res1 = tdinat + tsneg - assert res1[1] is NaT - res2 = tdinat + tspos - assert res2[1] is NaT - - def test_timedeltaindex_sub_timestamp_nat_masking(self): - tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) - - # tsneg.value < 0, tspos.value > 0 - tsneg = Timestamp('1950-01-01') - tspos = Timestamp('1980-01-01') - - res1 = tdinat - tsneg - assert res1[1] is NaT - res2 = tdinat - tspos - assert res2[1] is NaT - - def test_timedeltaindex_add_timestamp_overflow(self): - tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) - tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) - - # tsneg.value < 0, tspos.value > 0 - tsneg = Timestamp('1950-01-01') - tspos = Timestamp('1980-01-01') - - res1 = tdimax + tsneg - assert res1[1].value == Timedelta.max.value + tsneg.value - res2 = tdimin + tspos - assert res2[1].value == Timedelta.min.value + tspos.value - - with pytest.raises(OverflowError): - tdimax + tspos - - with pytest.raises(OverflowError): - tdimin + tsneg - - def test_timedeltaindex_add_timedelta_overflow(self): - tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) - tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) - - # tdpos.value > 0, tdneg.value < 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - with pytest.raises(OverflowError): - tdimax + tdpos - - res2 = tdimax + tdneg - assert res2[1].value == Timedelta.max.value + tdneg.value - res3 = tdimin + tdpos - assert res3[1].value == Timedelta.min.value + tdpos.value - - with pytest.raises(OverflowError): - tdimin + tdneg - - def test_timedeltaindex_sub_timedelta_overflow(self): - tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) - tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) - - # tdpos.value > 0, tdneg.value < 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - res1 = tdimax - tdpos - assert res1[1].value == Timedelta.max.value - tdpos.value - - with pytest.raises(OverflowError): - tdimax - tdneg - - with pytest.raises(OverflowError): - tdimin - tdpos - - res4 = tdimin - tdneg - assert res4[1].value == Timedelta.min.value - tdneg.value - - def test_datetimeindex_add_nat_masking(self): - # Checking for NaTs and checking that we don't get an OverflowError - dtinat = pd.to_datetime(['now', 'NaT']) - - # tdpos.value > 0, tdneg.value < 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - res1 = dtinat + tdpos - assert res1[1] is NaT - res2 = dtinat + tdneg - assert res2[1] is NaT - - def test_datetimeindex_sub_nat_masking(self): - # Checking for NaTs and checking that we don't get an OverflowError - dtinat = pd.to_datetime(['now', 'NaT']) - - # tdpos.value > 0, tdneg.value < 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - res1 = dtinat - tdpos - assert res1[1] is NaT - res2 = dtinat - tdneg - assert res2[1] is NaT - - def test_datetimeindex_add_timedelta_overflow(self): - dtimax = pd.to_datetime(['now', Timestamp.max]) - dtimin = pd.to_datetime(['now', Timestamp.min]) - - # tdpos.value < 0, tdneg.value > 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - with pytest.raises(OverflowError): - dtimax + tdpos - - res2 = dtimax + tdneg - assert res2[1].value == Timestamp.max.value + tdneg.value - - res3 = dtimin + tdpos - assert res3[1].value == Timestamp.min.value + tdpos.value - - with pytest.raises(OverflowError): - dtimin + tdneg - - def test_datetimeindex_sub_timedelta_overflow(self): - dtimax = pd.to_datetime(['now', Timestamp.max]) - dtimin = pd.to_datetime(['now', Timestamp.min]) - - # tdpos.value < 0, tdneg.value > 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - res1 = dtimax - tdpos - assert res1[1].value == Timestamp.max.value - tdpos.value - - with pytest.raises(OverflowError): - dtimax - tdneg - - with pytest.raises(OverflowError): - dtimin - tdpos - - res4 = dtimin - tdneg - assert res4[1].value == Timestamp.min.value - tdneg.value - - def test_datetimeindex_sub_timestamp_overflow(self): - dtimax = pd.to_datetime(['now', Timestamp.max]) - dtimin = pd.to_datetime(['now', Timestamp.min]) - - # tsneg.value < 0, tspos.value > 0 - tsneg = Timestamp('1950-01-01') - tspos = Timestamp('1980-01-01') - - with pytest.raises(OverflowError): - dtimax - tsneg - - res2 = dtimax - tspos - assert res2[1].value == Timestamp.max.value - tspos.value - - res3 = dtimin - tsneg - assert res3[1].value == Timestamp.min.value - tsneg.value - - with pytest.raises(OverflowError): - dtimin - tspos diff --git a/pandas/tests/indexes/test_datetimelike_arithmetic.py b/pandas/tests/indexes/test_datetimelike_arithmetic.py new file mode 100644 index 0000000000000..361f659cd3db0 --- /dev/null +++ b/pandas/tests/indexes/test_datetimelike_arithmetic.py @@ -0,0 +1,195 @@ +""" Test Matrix for arithmetic operations on DatetimeIndex, TimedeltaIndex, +and PeriodIndex +""" + +import pytest + +import pandas as pd +from pandas import Timestamp, Timedelta, NaT + + +class TestDatetimeLikeIndexArithmetic(object): + # GH17991 checking for overflows and NaT masking on arithmetic ops + + # TODO: Fill out the matrix of allowed arithmetic operations: + # - __rsub__, __radd__ + # - ops with scalars boxed in Index/Series/DataFrame/np.array + # - ops with scalars: + # NaT, Timestamp.min/max, Timedelta.min/max + # datetime, timedelta, date(?), + # relativedelta, + # np.datetime64, np.timedelta64, + # DateOffset, + # Period + # - timezone-aware variants + # - object-dtype, categorical dtype + # - PeriodIndex + # - consistency with .map(...) ? + + def test_timedeltaindex_add_timestamp_nat_masking(self): + tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) + + # tsneg.value < 0, tspos.value > 0 + tsneg = Timestamp('1950-01-01') + tspos = Timestamp('1980-01-01') + + res1 = tdinat + tsneg + assert res1[1] is NaT + res2 = tdinat + tspos + assert res2[1] is NaT + + def test_timedeltaindex_sub_timestamp_nat_masking(self): + tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) + + # tsneg.value < 0, tspos.value > 0 + tsneg = Timestamp('1950-01-01') + tspos = Timestamp('1980-01-01') + + res1 = tdinat - tsneg + assert res1[1] is NaT + res2 = tdinat - tspos + assert res2[1] is NaT + + def test_timedeltaindex_add_timestamp_overflow(self): + tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) + tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) + + # tsneg.value < 0, tspos.value > 0 + tsneg = Timestamp('1950-01-01') + tspos = Timestamp('1980-01-01') + + res1 = tdimax + tsneg + assert res1[1].value == Timedelta.max.value + tsneg.value + res2 = tdimin + tspos + assert res2[1].value == Timedelta.min.value + tspos.value + + with pytest.raises(OverflowError): + tdimax + tspos + + with pytest.raises(OverflowError): + tdimin + tsneg + + def test_timedeltaindex_add_timedelta_overflow(self): + tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) + tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) + + # tdpos.value > 0, tdneg.value < 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + with pytest.raises(OverflowError): + tdimax + tdpos + + res2 = tdimax + tdneg + assert res2[1].value == Timedelta.max.value + tdneg.value + res3 = tdimin + tdpos + assert res3[1].value == Timedelta.min.value + tdpos.value + + with pytest.raises(OverflowError): + tdimin + tdneg + + def test_timedeltaindex_sub_timedelta_overflow(self): + tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) + tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) + + # tdpos.value > 0, tdneg.value < 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + res1 = tdimax - tdpos + assert res1[1].value == Timedelta.max.value - tdpos.value + + with pytest.raises(OverflowError): + tdimax - tdneg + + with pytest.raises(OverflowError): + tdimin - tdpos + + res4 = tdimin - tdneg + assert res4[1].value == Timedelta.min.value - tdneg.value + + def test_datetimeindex_add_nat_masking(self): + # Checking for NaTs and checking that we don't get an OverflowError + dtinat = pd.to_datetime(['now', 'NaT']) + + # tdpos.value > 0, tdneg.value < 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + res1 = dtinat + tdpos + assert res1[1] is NaT + res2 = dtinat + tdneg + assert res2[1] is NaT + + def test_datetimeindex_sub_nat_masking(self): + # Checking for NaTs and checking that we don't get an OverflowError + dtinat = pd.to_datetime(['now', 'NaT']) + + # tdpos.value > 0, tdneg.value < 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + res1 = dtinat - tdpos + assert res1[1] is NaT + res2 = dtinat - tdneg + assert res2[1] is NaT + + def test_datetimeindex_add_timedelta_overflow(self): + dtimax = pd.to_datetime(['now', Timestamp.max]) + dtimin = pd.to_datetime(['now', Timestamp.min]) + + # tdpos.value < 0, tdneg.value > 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + with pytest.raises(OverflowError): + dtimax + tdpos + + res2 = dtimax + tdneg + assert res2[1].value == Timestamp.max.value + tdneg.value + + res3 = dtimin + tdpos + assert res3[1].value == Timestamp.min.value + tdpos.value + + with pytest.raises(OverflowError): + dtimin + tdneg + + def test_datetimeindex_sub_timedelta_overflow(self): + dtimax = pd.to_datetime(['now', Timestamp.max]) + dtimin = pd.to_datetime(['now', Timestamp.min]) + + # tdpos.value < 0, tdneg.value > 0 + tdpos = Timedelta('1h') + tdneg = Timedelta('-1h') + + res1 = dtimax - tdpos + assert res1[1].value == Timestamp.max.value - tdpos.value + + with pytest.raises(OverflowError): + dtimax - tdneg + + with pytest.raises(OverflowError): + dtimin - tdpos + + res4 = dtimin - tdneg + assert res4[1].value == Timestamp.min.value - tdneg.value + + def test_datetimeindex_sub_timestamp_overflow(self): + dtimax = pd.to_datetime(['now', Timestamp.max]) + dtimin = pd.to_datetime(['now', Timestamp.min]) + + # tsneg.value < 0, tspos.value > 0 + tsneg = Timestamp('1950-01-01') + tspos = Timestamp('1980-01-01') + + with pytest.raises(OverflowError): + dtimax - tsneg + + res2 = dtimax - tspos + assert res2[1].value == Timestamp.max.value - tspos.value + + res3 = dtimin - tsneg + assert res3[1].value == Timestamp.min.value - tsneg.value + + with pytest.raises(OverflowError): + dtimin - tspos From de663a1837faee4e61ea8d7ead6768e0a094e1e7 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 12:37:31 -0700 Subject: [PATCH 05/13] remove invalid test --- pandas/tests/indexes/test_datetimelike_arithmetic.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pandas/tests/indexes/test_datetimelike_arithmetic.py b/pandas/tests/indexes/test_datetimelike_arithmetic.py index 361f659cd3db0..b5f625ffaa13f 100644 --- a/pandas/tests/indexes/test_datetimelike_arithmetic.py +++ b/pandas/tests/indexes/test_datetimelike_arithmetic.py @@ -38,18 +38,6 @@ def test_timedeltaindex_add_timestamp_nat_masking(self): res2 = tdinat + tspos assert res2[1] is NaT - def test_timedeltaindex_sub_timestamp_nat_masking(self): - tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) - - # tsneg.value < 0, tspos.value > 0 - tsneg = Timestamp('1950-01-01') - tspos = Timestamp('1980-01-01') - - res1 = tdinat - tsneg - assert res1[1] is NaT - res2 = tdinat - tspos - assert res2[1] is NaT - def test_timedeltaindex_add_timestamp_overflow(self): tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) From 183e8d0b3be7e290aa144d90fd4acf49ea093aca Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 13:12:11 -0700 Subject: [PATCH 06/13] Move towards parameterized tests; fix for np.datetime64 arith #7996 --- pandas/core/indexes/datetimelike.py | 2 +- pandas/core/indexes/datetimes.py | 2 +- .../indexes/test_datetimelike_arithmetic.py | 257 ++++++++++-------- 3 files changed, 141 insertions(+), 120 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 71de6c7c3e8cf..4e9b2b9a2e922 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -681,7 +681,7 @@ def __sub__(self, other): return self._add_delta(-other) elif is_integer(other): return self.shift(-other) - elif isinstance(other, datetime): + elif isinstance(other, (datetime, np.datetime64)): return self._sub_datelike(other) elif isinstance(other, Period): return self._sub_period(other) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 977fb45d5d778..15d78985bd41e 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -768,7 +768,7 @@ def _sub_datelike(self, other): raise TypeError("DatetimeIndex subtraction must have the same " "timezones or no timezones") result = self._sub_datelike_dti(other) - elif isinstance(other, datetime): + elif isinstance(other, (datetime, np.datetime64)): other = Timestamp(other) if other is libts.NaT: result = self._nat_new(box=False) diff --git a/pandas/tests/indexes/test_datetimelike_arithmetic.py b/pandas/tests/indexes/test_datetimelike_arithmetic.py index b5f625ffaa13f..1caaef2817c89 100644 --- a/pandas/tests/indexes/test_datetimelike_arithmetic.py +++ b/pandas/tests/indexes/test_datetimelike_arithmetic.py @@ -1,12 +1,45 @@ """ Test Matrix for arithmetic operations on DatetimeIndex, TimedeltaIndex, and PeriodIndex """ +from datetime import datetime, timedelta import pytest +import numpy as np import pandas as pd from pandas import Timestamp, Timedelta, NaT +tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) +tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) +tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) + +dtinat = pd.to_datetime(['now', 'NaT']) +dtimax = pd.to_datetime(['now', Timestamp.max]) +dtimin = pd.to_datetime(['now', Timestamp.min]) + + +tspos = Timestamp('1980-01-01') +ts_pos_variants = [tspos, + datetime(1980, 1, 1), + np.datetime64('1980-01-01').astype('M8[ns]'), + np.datetime64('1980-01-01').astype('M8[D]')] + +tsneg = Timestamp('1950-01-01') +ts_neg_variants = [tsneg, + datetime(1950, 1, 1), + np.datetime64('1950-01-01').astype('M8[ns]'), + np.datetime64('1950-01-01').astype('M8[D]')] + +tdpos = Timedelta('1h') +td_pos_variants = [tdpos, + tdpos.to_pytimedelta(), + tdpos.to_timedelta64()] + +tdneg = Timedelta('-1h') +td_neg_variants = [tdneg, + tdneg.to_pytimedelta(), + tdneg.to_timedelta64()] + class TestDatetimeLikeIndexArithmetic(object): # GH17991 checking for overflows and NaT masking on arithmetic ops @@ -25,159 +58,147 @@ class TestDatetimeLikeIndexArithmetic(object): # - object-dtype, categorical dtype # - PeriodIndex # - consistency with .map(...) ? + # - versions with near-min/max values def test_timedeltaindex_add_timestamp_nat_masking(self): - tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) - - # tsneg.value < 0, tspos.value > 0 - tsneg = Timestamp('1950-01-01') - tspos = Timestamp('1980-01-01') + for variant in ts_neg_variants: + res = tdinat + variant + assert res[1] is NaT - res1 = tdinat + tsneg - assert res1[1] is NaT - res2 = tdinat + tspos - assert res2[1] is NaT + for variant in ts_pos_variants: + res = tdinat + variant + assert res[1] is NaT def test_timedeltaindex_add_timestamp_overflow(self): - tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) - tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) - - # tsneg.value < 0, tspos.value > 0 - tsneg = Timestamp('1950-01-01') - tspos = Timestamp('1980-01-01') + expected = Timedelta.max.value + tsneg.value + for variant in ts_neg_variants: + res = tdimax + variant + assert res[1].value == expected - res1 = tdimax + tsneg - assert res1[1].value == Timedelta.max.value + tsneg.value - res2 = tdimin + tspos - assert res2[1].value == Timedelta.min.value + tspos.value + expected = Timedelta.min.value + tspos.value + for variant in ts_pos_variants: + res = tdimin + variant + assert res[1].value == expected - with pytest.raises(OverflowError): - tdimax + tspos + for variant in ts_pos_variants: + with pytest.raises(OverflowError): + tdimax + variant - with pytest.raises(OverflowError): - tdimin + tsneg + for variant in ts_neg_variants: + with pytest.raises(OverflowError): + tdimin + variant def test_timedeltaindex_add_timedelta_overflow(self): - tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) - tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) + for variant in td_pos_variants: + with pytest.raises(OverflowError): + tdimax + variant - # tdpos.value > 0, tdneg.value < 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') + expected = Timedelta.max.value + tdneg.value + for variant in td_neg_variants: + res = tdimax + variant + assert res[1].value == expected - with pytest.raises(OverflowError): - tdimax + tdpos + expected = Timedelta.min.value + tdpos.value + for variant in td_pos_variants: + res = tdimin + variant + assert res[1].value == expected - res2 = tdimax + tdneg - assert res2[1].value == Timedelta.max.value + tdneg.value - res3 = tdimin + tdpos - assert res3[1].value == Timedelta.min.value + tdpos.value - - with pytest.raises(OverflowError): - tdimin + tdneg + for variant in td_neg_variants: + with pytest.raises(OverflowError): + tdimin + variant def test_timedeltaindex_sub_timedelta_overflow(self): - tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) - tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) - - # tdpos.value > 0, tdneg.value < 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - res1 = tdimax - tdpos - assert res1[1].value == Timedelta.max.value - tdpos.value + expected = Timedelta.max.value - tdpos.value + for variant in td_pos_variants: + res1 = tdimax - variant + assert res1[1].value == expected - with pytest.raises(OverflowError): - tdimax - tdneg + for variant in td_neg_variants: + with pytest.raises(OverflowError): + tdimax - variant - with pytest.raises(OverflowError): - tdimin - tdpos + for variant in td_pos_variants: + with pytest.raises(OverflowError): + tdimin - variant - res4 = tdimin - tdneg - assert res4[1].value == Timedelta.min.value - tdneg.value + expected = Timedelta.min.value - tdneg.value + for variant in td_neg_variants: + res = tdimin - variant + assert res[1].value == expected def test_datetimeindex_add_nat_masking(self): # Checking for NaTs and checking that we don't get an OverflowError - dtinat = pd.to_datetime(['now', 'NaT']) + for variant in td_pos_variants: + res = dtinat + variant + assert res[1] is NaT - # tdpos.value > 0, tdneg.value < 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - res1 = dtinat + tdpos - assert res1[1] is NaT - res2 = dtinat + tdneg - assert res2[1] is NaT + for variant in td_neg_variants: + res = dtinat + variant + assert res[1] is NaT def test_datetimeindex_sub_nat_masking(self): # Checking for NaTs and checking that we don't get an OverflowError - dtinat = pd.to_datetime(['now', 'NaT']) - - # tdpos.value > 0, tdneg.value < 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') + for variant in td_pos_variants: + res = dtinat - variant + assert res[1] is NaT - res1 = dtinat - tdpos - assert res1[1] is NaT - res2 = dtinat - tdneg - assert res2[1] is NaT + for variant in td_neg_variants: + res = dtinat - variant + assert res[1] is NaT def test_datetimeindex_add_timedelta_overflow(self): - dtimax = pd.to_datetime(['now', Timestamp.max]) - dtimin = pd.to_datetime(['now', Timestamp.min]) - - # tdpos.value < 0, tdneg.value > 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') + for variant in td_pos_variants: + with pytest.raises(OverflowError): + dtimax + variant - with pytest.raises(OverflowError): - dtimax + tdpos + expected = Timestamp.max.value + tdneg.value + for variant in td_neg_variants: + res = dtimax + variant + assert res[1].value == expected - res2 = dtimax + tdneg - assert res2[1].value == Timestamp.max.value + tdneg.value + expected = Timestamp.min.value + tdpos.value + for variant in td_pos_variants: + res = dtimin + variant + assert res[1].value == expected - res3 = dtimin + tdpos - assert res3[1].value == Timestamp.min.value + tdpos.value - - with pytest.raises(OverflowError): - dtimin + tdneg + for variant in td_neg_variants: + with pytest.raises(OverflowError): + dtimin + variant def test_datetimeindex_sub_timedelta_overflow(self): - dtimax = pd.to_datetime(['now', Timestamp.max]) - dtimin = pd.to_datetime(['now', Timestamp.min]) - - # tdpos.value < 0, tdneg.value > 0 - tdpos = Timedelta('1h') - tdneg = Timedelta('-1h') - - res1 = dtimax - tdpos - assert res1[1].value == Timestamp.max.value - tdpos.value + expected = Timestamp.max.value - tdpos.value + for variant in td_pos_variants: + res = dtimax - variant + assert res[1].value == expected - with pytest.raises(OverflowError): - dtimax - tdneg + for variant in td_neg_variants: + with pytest.raises(OverflowError): + dtimax - variant - with pytest.raises(OverflowError): - dtimin - tdpos + for variant in td_pos_variants: + with pytest.raises(OverflowError): + dtimin - variant - res4 = dtimin - tdneg - assert res4[1].value == Timestamp.min.value - tdneg.value + expected = Timestamp.min.value - tdneg.value + for variant in td_neg_variants: + res = dtimin - variant + assert res[1].value == expected def test_datetimeindex_sub_timestamp_overflow(self): - dtimax = pd.to_datetime(['now', Timestamp.max]) - dtimin = pd.to_datetime(['now', Timestamp.min]) - - # tsneg.value < 0, tspos.value > 0 - tsneg = Timestamp('1950-01-01') - tspos = Timestamp('1980-01-01') - - with pytest.raises(OverflowError): - dtimax - tsneg - - res2 = dtimax - tspos - assert res2[1].value == Timestamp.max.value - tspos.value - - res3 = dtimin - tsneg - assert res3[1].value == Timestamp.min.value - tsneg.value - - with pytest.raises(OverflowError): - dtimin - tspos + for variant in ts_neg_variants: + with pytest.raises(OverflowError): + dtimax - variant + + expected = Timestamp.max.value - tspos.value + for variant in ts_pos_variants: + res = dtimax - variant + assert res[1].value == expected + + expected = Timestamp.min.value - tsneg.value + for variant in ts_neg_variants: + res = dtimin - variant + assert res[1].value == expected + + for variant in ts_pos_variants: + with pytest.raises(OverflowError): + dtimin - variant From 75a403c4e5c5a52bf50d33ef256bd2de4a63194a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 13:20:47 -0700 Subject: [PATCH 07/13] test cases for datetime64/timedelta64 with different units --- .../indexes/test_datetimelike_arithmetic.py | 78 +++++++++++++------ 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/pandas/tests/indexes/test_datetimelike_arithmetic.py b/pandas/tests/indexes/test_datetimelike_arithmetic.py index 1caaef2817c89..494b3c28a8f79 100644 --- a/pandas/tests/indexes/test_datetimelike_arithmetic.py +++ b/pandas/tests/indexes/test_datetimelike_arithmetic.py @@ -9,6 +9,22 @@ import pandas as pd from pandas import Timestamp, Timedelta, NaT +# TODO: Fill out the matrix of allowed arithmetic operations: +# - __rsub__, __radd__ +# - ops with scalars boxed in Index/Series/DataFrame/np.array +# - ops with scalars: +# NaT, Timestamp.min/max, Timedelta.min/max +# datetime, timedelta, date(?), +# relativedelta, +# np.datetime64, np.timedelta64, +# DateOffset, +# Period +# - timezone-aware variants +# - object-dtype, categorical dtype +# - PeriodIndex +# - consistency with .map(...) ? +# - versions with near-min/max values + tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) @@ -21,44 +37,50 @@ tspos = Timestamp('1980-01-01') ts_pos_variants = [tspos, datetime(1980, 1, 1), - np.datetime64('1980-01-01').astype('M8[ns]'), - np.datetime64('1980-01-01').astype('M8[D]')] + np.datetime64('1980-01-01').astype('datetime64[ns]'), + np.datetime64('1980-01-01').astype('datetime64[D]')] tsneg = Timestamp('1950-01-01') ts_neg_variants = [tsneg, datetime(1950, 1, 1), - np.datetime64('1950-01-01').astype('M8[ns]'), - np.datetime64('1950-01-01').astype('M8[D]')] + np.datetime64('1950-01-01').astype('datetime64[ns]'), + np.datetime64('1950-01-01').astype('datetime64[D]')] tdpos = Timedelta('1h') td_pos_variants = [tdpos, tdpos.to_pytimedelta(), - tdpos.to_timedelta64()] + tdpos.to_timedelta64().astype('timedelta64[ns]'), + tdpos.to_timedelta64().astype('timedelta64[h]')] tdneg = Timedelta('-1h') td_neg_variants = [tdneg, tdneg.to_pytimedelta(), - tdneg.to_timedelta64()] + tdneg.to_timedelta64().astype('timedelta64[ns]'), + tdneg.to_timedelta64().astype('timedelta64[h]')] class TestDatetimeLikeIndexArithmetic(object): # GH17991 checking for overflows and NaT masking on arithmetic ops - # TODO: Fill out the matrix of allowed arithmetic operations: - # - __rsub__, __radd__ - # - ops with scalars boxed in Index/Series/DataFrame/np.array - # - ops with scalars: - # NaT, Timestamp.min/max, Timedelta.min/max - # datetime, timedelta, date(?), - # relativedelta, - # np.datetime64, np.timedelta64, - # DateOffset, - # Period - # - timezone-aware variants - # - object-dtype, categorical dtype - # - PeriodIndex - # - consistency with .map(...) ? - # - versions with near-min/max values + def test_datetimeindex_add_timedelta_nat_masking(self): + # Checking for NaTs and checking that we don't get an OverflowError + for variant in td_pos_variants: + res = tdinat + variant + assert res[1] is NaT + + for variant in td_neg_variants: + res = tdinat + variant + assert res[1] is NaT + + def test_datetimeindex_sub_timedelta_nat_masking(self): + # Checking for NaTs and checking that we don't get an OverflowError + for variant in td_pos_variants: + res = tdinat - variant + assert res[1] is NaT + + for variant in td_neg_variants: + res = tdinat - variant + assert res[1] is NaT def test_timedeltaindex_add_timestamp_nat_masking(self): for variant in ts_neg_variants: @@ -126,7 +148,7 @@ def test_timedeltaindex_sub_timedelta_overflow(self): res = tdimin - variant assert res[1].value == expected - def test_datetimeindex_add_nat_masking(self): + def test_datetimeindex_add_timedelta_nat_masking(self): # Checking for NaTs and checking that we don't get an OverflowError for variant in td_pos_variants: res = dtinat + variant @@ -136,7 +158,7 @@ def test_datetimeindex_add_nat_masking(self): res = dtinat + variant assert res[1] is NaT - def test_datetimeindex_sub_nat_masking(self): + def test_datetimeindex_sub_timedelta_nat_masking(self): # Checking for NaTs and checking that we don't get an OverflowError for variant in td_pos_variants: res = dtinat - variant @@ -146,6 +168,16 @@ def test_datetimeindex_sub_nat_masking(self): res = dtinat - variant assert res[1] is NaT + def test_datetimeindex_sub_timestamp_nat_masking(self): + # Checking for NaTs and checking that we don't get an OverflowError + for variant in ts_pos_variants: + res = dtinat - variant + assert res[1] is NaT + + for variant in ts_neg_variants: + res = dtinat - variant + assert res[1] is NaT + def test_datetimeindex_add_timedelta_overflow(self): for variant in td_pos_variants: with pytest.raises(OverflowError): From ff72d983e40633e962c2118c5da7fa5dbd94bac1 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 13:48:24 -0700 Subject: [PATCH 08/13] Remove new test file; move two currently-failing tests to existing modules --- .../tests/indexes/datetimes/test_datetime.py | 34 +++ .../indexes/test_datetimelike_arithmetic.py | 236 ------------------ pandas/tests/indexes/timedeltas/test_ops.py | 20 ++ 3 files changed, 54 insertions(+), 236 deletions(-) delete mode 100644 pandas/tests/indexes/test_datetimelike_arithmetic.py diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py index 8d9ac59cf9883..0faae7edc0906 100644 --- a/pandas/tests/indexes/datetimes/test_datetime.py +++ b/pandas/tests/indexes/datetimes/test_datetime.py @@ -447,6 +447,40 @@ def f(): t - offset pytest.raises(OverflowError, f) + def test_datetimeindex_sub_timestamp_overflow(self): + tdimax = pd.to_timedelta(['24658 days 11:15:00', pd.Timedelta.max]) + tdimin = pd.to_timedelta(['24658 days 11:15:00', pd.Timedelta.min]) + + tsneg = Timestamp('1950-01-01') + ts_neg_variants = [tsneg, + tsneg.to_pydatetime(), + tsneg.to_datetime64().astype('datetime64[ns]'), + tsneg.to_datetime64().astype('datetime64[D]')] + + tspos = Timestamp('1980-01-01') + ts_pos_variants = [tspos, + tspos.to_pydatetime(), + tspos.to_datetime64().astype('datetime64[ns]'), + tspos.to_datetime64().astype('datetime64[D]')] + + for variant in ts_neg_variants: + with pytest.raises(OverflowError): + dtimax - variant + + expected = pd.Timestamp.max.value - tspos.value + for variant in ts_pos_variants: + res = dtimax - variant + assert res[1].value == expected + + expected = pd.Timestamp.min.value - tsneg.value + for variant in ts_neg_variants: + res = dtimin - variant + assert res[1].value == expected + + for variant in ts_pos_variants: + with pytest.raises(OverflowError): + dtimin - variant + def test_get_duplicates(self): idx = DatetimeIndex(['2000-01-01', '2000-01-02', '2000-01-02', '2000-01-03', '2000-01-03', '2000-01-04']) diff --git a/pandas/tests/indexes/test_datetimelike_arithmetic.py b/pandas/tests/indexes/test_datetimelike_arithmetic.py deleted file mode 100644 index 494b3c28a8f79..0000000000000 --- a/pandas/tests/indexes/test_datetimelike_arithmetic.py +++ /dev/null @@ -1,236 +0,0 @@ -""" Test Matrix for arithmetic operations on DatetimeIndex, TimedeltaIndex, -and PeriodIndex -""" -from datetime import datetime, timedelta - -import pytest -import numpy as np - -import pandas as pd -from pandas import Timestamp, Timedelta, NaT - -# TODO: Fill out the matrix of allowed arithmetic operations: -# - __rsub__, __radd__ -# - ops with scalars boxed in Index/Series/DataFrame/np.array -# - ops with scalars: -# NaT, Timestamp.min/max, Timedelta.min/max -# datetime, timedelta, date(?), -# relativedelta, -# np.datetime64, np.timedelta64, -# DateOffset, -# Period -# - timezone-aware variants -# - object-dtype, categorical dtype -# - PeriodIndex -# - consistency with .map(...) ? -# - versions with near-min/max values - -tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) -tdimax = pd.to_timedelta(['24658 days 11:15:00', Timedelta.max]) -tdimin = pd.to_timedelta(['24658 days 11:15:00', Timedelta.min]) - -dtinat = pd.to_datetime(['now', 'NaT']) -dtimax = pd.to_datetime(['now', Timestamp.max]) -dtimin = pd.to_datetime(['now', Timestamp.min]) - - -tspos = Timestamp('1980-01-01') -ts_pos_variants = [tspos, - datetime(1980, 1, 1), - np.datetime64('1980-01-01').astype('datetime64[ns]'), - np.datetime64('1980-01-01').astype('datetime64[D]')] - -tsneg = Timestamp('1950-01-01') -ts_neg_variants = [tsneg, - datetime(1950, 1, 1), - np.datetime64('1950-01-01').astype('datetime64[ns]'), - np.datetime64('1950-01-01').astype('datetime64[D]')] - -tdpos = Timedelta('1h') -td_pos_variants = [tdpos, - tdpos.to_pytimedelta(), - tdpos.to_timedelta64().astype('timedelta64[ns]'), - tdpos.to_timedelta64().astype('timedelta64[h]')] - -tdneg = Timedelta('-1h') -td_neg_variants = [tdneg, - tdneg.to_pytimedelta(), - tdneg.to_timedelta64().astype('timedelta64[ns]'), - tdneg.to_timedelta64().astype('timedelta64[h]')] - - -class TestDatetimeLikeIndexArithmetic(object): - # GH17991 checking for overflows and NaT masking on arithmetic ops - - def test_datetimeindex_add_timedelta_nat_masking(self): - # Checking for NaTs and checking that we don't get an OverflowError - for variant in td_pos_variants: - res = tdinat + variant - assert res[1] is NaT - - for variant in td_neg_variants: - res = tdinat + variant - assert res[1] is NaT - - def test_datetimeindex_sub_timedelta_nat_masking(self): - # Checking for NaTs and checking that we don't get an OverflowError - for variant in td_pos_variants: - res = tdinat - variant - assert res[1] is NaT - - for variant in td_neg_variants: - res = tdinat - variant - assert res[1] is NaT - - def test_timedeltaindex_add_timestamp_nat_masking(self): - for variant in ts_neg_variants: - res = tdinat + variant - assert res[1] is NaT - - for variant in ts_pos_variants: - res = tdinat + variant - assert res[1] is NaT - - def test_timedeltaindex_add_timestamp_overflow(self): - expected = Timedelta.max.value + tsneg.value - for variant in ts_neg_variants: - res = tdimax + variant - assert res[1].value == expected - - expected = Timedelta.min.value + tspos.value - for variant in ts_pos_variants: - res = tdimin + variant - assert res[1].value == expected - - for variant in ts_pos_variants: - with pytest.raises(OverflowError): - tdimax + variant - - for variant in ts_neg_variants: - with pytest.raises(OverflowError): - tdimin + variant - - def test_timedeltaindex_add_timedelta_overflow(self): - for variant in td_pos_variants: - with pytest.raises(OverflowError): - tdimax + variant - - expected = Timedelta.max.value + tdneg.value - for variant in td_neg_variants: - res = tdimax + variant - assert res[1].value == expected - - expected = Timedelta.min.value + tdpos.value - for variant in td_pos_variants: - res = tdimin + variant - assert res[1].value == expected - - for variant in td_neg_variants: - with pytest.raises(OverflowError): - tdimin + variant - - def test_timedeltaindex_sub_timedelta_overflow(self): - expected = Timedelta.max.value - tdpos.value - for variant in td_pos_variants: - res1 = tdimax - variant - assert res1[1].value == expected - - for variant in td_neg_variants: - with pytest.raises(OverflowError): - tdimax - variant - - for variant in td_pos_variants: - with pytest.raises(OverflowError): - tdimin - variant - - expected = Timedelta.min.value - tdneg.value - for variant in td_neg_variants: - res = tdimin - variant - assert res[1].value == expected - - def test_datetimeindex_add_timedelta_nat_masking(self): - # Checking for NaTs and checking that we don't get an OverflowError - for variant in td_pos_variants: - res = dtinat + variant - assert res[1] is NaT - - for variant in td_neg_variants: - res = dtinat + variant - assert res[1] is NaT - - def test_datetimeindex_sub_timedelta_nat_masking(self): - # Checking for NaTs and checking that we don't get an OverflowError - for variant in td_pos_variants: - res = dtinat - variant - assert res[1] is NaT - - for variant in td_neg_variants: - res = dtinat - variant - assert res[1] is NaT - - def test_datetimeindex_sub_timestamp_nat_masking(self): - # Checking for NaTs and checking that we don't get an OverflowError - for variant in ts_pos_variants: - res = dtinat - variant - assert res[1] is NaT - - for variant in ts_neg_variants: - res = dtinat - variant - assert res[1] is NaT - - def test_datetimeindex_add_timedelta_overflow(self): - for variant in td_pos_variants: - with pytest.raises(OverflowError): - dtimax + variant - - expected = Timestamp.max.value + tdneg.value - for variant in td_neg_variants: - res = dtimax + variant - assert res[1].value == expected - - expected = Timestamp.min.value + tdpos.value - for variant in td_pos_variants: - res = dtimin + variant - assert res[1].value == expected - - for variant in td_neg_variants: - with pytest.raises(OverflowError): - dtimin + variant - - def test_datetimeindex_sub_timedelta_overflow(self): - expected = Timestamp.max.value - tdpos.value - for variant in td_pos_variants: - res = dtimax - variant - assert res[1].value == expected - - for variant in td_neg_variants: - with pytest.raises(OverflowError): - dtimax - variant - - for variant in td_pos_variants: - with pytest.raises(OverflowError): - dtimin - variant - - expected = Timestamp.min.value - tdneg.value - for variant in td_neg_variants: - res = dtimin - variant - assert res[1].value == expected - - def test_datetimeindex_sub_timestamp_overflow(self): - for variant in ts_neg_variants: - with pytest.raises(OverflowError): - dtimax - variant - - expected = Timestamp.max.value - tspos.value - for variant in ts_pos_variants: - res = dtimax - variant - assert res[1].value == expected - - expected = Timestamp.min.value - tsneg.value - for variant in ts_neg_variants: - res = dtimin - variant - assert res[1].value == expected - - for variant in ts_pos_variants: - with pytest.raises(OverflowError): - dtimin - variant diff --git a/pandas/tests/indexes/timedeltas/test_ops.py b/pandas/tests/indexes/timedeltas/test_ops.py index f4f669ee1d087..07a4f1c96f3f6 100644 --- a/pandas/tests/indexes/timedeltas/test_ops.py +++ b/pandas/tests/indexes/timedeltas/test_ops.py @@ -1282,3 +1282,23 @@ def test_add_overflow(self): result = (to_timedelta([pd.NaT, '5 days', '1 hours']) + to_timedelta(['7 seconds', pd.NaT, '4 hours'])) tm.assert_index_equal(result, exp) + + def test_timedeltaindex_add_timestamp_nat_masking(self): + # GH17991 checking for overflow-masking with NaT + tdinat = pd.to_timedelta(['24658 days 11:15:00', 'NaT']) + + tsneg = Timestamp('1950-01-01') + ts_neg_variants = [tsneg, + tsneg.to_datetime(), + tsneg.to_datetime64().astype('datetime64[ns]') + tsneg.to_datetime64().astype('datetime64[D]')] + + tspos = Timestamp('1980-01-01') + ts_pos_variants = [tspos, + tspos.to_pydatetime(), + tspos.to_datetime64().astype('datetime64[ns]'), + tspos.to_datetime64().astype('datetime64[D]')] + + for variant in ts_neg_variants + ts_pos_variants: + res = tdinat + variant + assert res[1] is pd.NaT From ac246b1bb33dcc0bf443680ad3c526fc0cbbd9df Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 16:42:04 -0700 Subject: [PATCH 09/13] typo fixup --- pandas/tests/indexes/timedeltas/test_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/timedeltas/test_ops.py b/pandas/tests/indexes/timedeltas/test_ops.py index 07a4f1c96f3f6..63063b32120a2 100644 --- a/pandas/tests/indexes/timedeltas/test_ops.py +++ b/pandas/tests/indexes/timedeltas/test_ops.py @@ -1290,7 +1290,7 @@ def test_timedeltaindex_add_timestamp_nat_masking(self): tsneg = Timestamp('1950-01-01') ts_neg_variants = [tsneg, tsneg.to_datetime(), - tsneg.to_datetime64().astype('datetime64[ns]') + tsneg.to_datetime64().astype('datetime64[ns]'), tsneg.to_datetime64().astype('datetime64[D]')] tspos = Timestamp('1980-01-01') From 05254e294f459869096344a43022d0638064bc43 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 21:11:43 -0700 Subject: [PATCH 10/13] fixup NameErrors --- pandas/tests/indexes/datetimes/test_datetime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py index 0faae7edc0906..6621f9902279a 100644 --- a/pandas/tests/indexes/datetimes/test_datetime.py +++ b/pandas/tests/indexes/datetimes/test_datetime.py @@ -448,8 +448,8 @@ def f(): pytest.raises(OverflowError, f) def test_datetimeindex_sub_timestamp_overflow(self): - tdimax = pd.to_timedelta(['24658 days 11:15:00', pd.Timedelta.max]) - tdimin = pd.to_timedelta(['24658 days 11:15:00', pd.Timedelta.min]) + dtimax = pd.to_datetime(['now', pd.Timestamp.max]) + dtimin = pd.to_datetime(['now', pd.Timestamp.min]) tsneg = Timestamp('1950-01-01') ts_neg_variants = [tsneg, From f048ffaa6d277375e422aafa6b9b925a4ca1f7e4 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 30 Oct 2017 10:14:55 -0700 Subject: [PATCH 11/13] to_datetime-->to_pydatetime; fixes unrelated test failure --- pandas/tests/indexes/timedeltas/test_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/timedeltas/test_ops.py b/pandas/tests/indexes/timedeltas/test_ops.py index 63063b32120a2..3cf56dc5115c2 100644 --- a/pandas/tests/indexes/timedeltas/test_ops.py +++ b/pandas/tests/indexes/timedeltas/test_ops.py @@ -1289,7 +1289,7 @@ def test_timedeltaindex_add_timestamp_nat_masking(self): tsneg = Timestamp('1950-01-01') ts_neg_variants = [tsneg, - tsneg.to_datetime(), + tsneg.to_pydatetime(), tsneg.to_datetime64().astype('datetime64[ns]'), tsneg.to_datetime64().astype('datetime64[D]')] From 7be598422f47cd7e86f7c07ac44f620bc8cde7d1 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 30 Oct 2017 18:06:17 -0700 Subject: [PATCH 12/13] whatsnew notes --- doc/source/whatsnew/v0.21.1.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/whatsnew/v0.21.1.txt b/doc/source/whatsnew/v0.21.1.txt index 03a8e83a713f0..33d1916778fb2 100644 --- a/doc/source/whatsnew/v0.21.1.txt +++ b/doc/source/whatsnew/v0.21.1.txt @@ -57,6 +57,8 @@ Documentation Changes Bug Fixes ~~~~~~~~~ - Bug in ``DataFrame.resample(...).apply(...)`` when there is a callable that returns different columns (:issue:`15169`) +- Bug in :class:`TimedeltaIndex` subtraction could incorrectly overflow when `NaT` is present (:issue:`17791`) +- Bug in :class:`DatetimeIndex` subtraction could fail to overflow (:issue:`18020`) Conversion ^^^^^^^^^^ From 24dd75b05628379956e377ebd7d9acfc1fdb600d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 4 Nov 2017 11:53:37 -0700 Subject: [PATCH 13/13] clarify todo note per reviewre suggestion --- doc/source/whatsnew/v0.21.1.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.21.1.txt b/doc/source/whatsnew/v0.21.1.txt index 67f8768cc7c18..49be2c8581ef1 100644 --- a/doc/source/whatsnew/v0.21.1.txt +++ b/doc/source/whatsnew/v0.21.1.txt @@ -57,8 +57,8 @@ Documentation Changes Bug Fixes ~~~~~~~~~ - Bug in ``DataFrame.resample(...).apply(...)`` when there is a callable that returns different columns (:issue:`15169`) -- Bug in :class:`TimedeltaIndex` subtraction could incorrectly overflow when `NaT` is present (:issue:`17791`) -- Bug in :class:`DatetimeIndex` subtraction could fail to overflow (:issue:`18020`) +- Bug in :class:`TimedeltaIndex` subtraction could incorrectly overflow when ``NaT`` is present (:issue:`17791`) +- Bug in :class:`DatetimeIndex` subtracting datetimelike from DatetimeIndex could fail to overflow (:issue:`18020`) Conversion ^^^^^^^^^^