From 764bc8c30611e7213aa91f7186cb0d7bc03cf53a Mon Sep 17 00:00:00 2001 From: Anton Lodder Date: Tue, 2 Mar 2021 20:59:01 -0500 Subject: [PATCH 1/3] BUG: cannot subtract Timestamp with different timezones (#31793) --- doc/source/whatsnew/v1.4.0.rst | 1 + pandas/_libs/tslibs/timestamps.pyx | 8 +-- pandas/core/arrays/datetimes.py | 21 +++---- pandas/tests/arithmetic/test_datetime64.py | 63 +++++++++++++++++-- pandas/tests/arithmetic/test_timedelta64.py | 12 +--- .../tests/scalar/timestamp/test_arithmetic.py | 34 +++++++++- 6 files changed, 108 insertions(+), 31 deletions(-) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index 241c005c400cd..1c1415255bf89 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -715,6 +715,7 @@ Timezones ^^^^^^^^^ - Bug in :func:`to_datetime` with ``infer_datetime_format=True`` failing to parse zero UTC offset (``Z``) correctly (:issue:`41047`) - Bug in :meth:`Series.dt.tz_convert` resetting index in a :class:`Series` with :class:`CategoricalIndex` (:issue:`43080`) +- Bug in ``Timestamp`` and ``DatetimeIndex`` incorrectly raising a ``TypeError`` when subtracting two timezone-aware objects with mismatched timezones (:issue:`31793`) - Numeric diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 307b6f2949881..304ac9405c5e1 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -342,10 +342,10 @@ cdef class _Timestamp(ABCTimestamp): else: self = type(other)(self) - # validate tz's - if not tz_compare(self.tzinfo, other.tzinfo): - raise TypeError("Timestamp subtraction must have the " - "same timezones or no timezones") + if (self.tzinfo is None) ^ (other.tzinfo is None): + raise TypeError( + "Cannot subtract tz-naive and tz-aware datetime-like objects." + ) # scalar Timestamp/datetime - Timestamp/datetime -> yields a # Timedelta diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index b3a1a4d342355..647698f472978 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -728,12 +728,11 @@ def _sub_datetime_arraylike(self, other): assert is_datetime64_dtype(other) other = type(self)(other) - if not self._has_same_tz(other): - # require tz compat - raise TypeError( - f"{type(self).__name__} subtraction must have the same " - "timezones or no timezones" - ) + try: + self._assert_tzawareness_compat(other) + except TypeError as error: + new_message = str(error).replace("compare", "subtract") + raise type(error)(new_message) from error self_i8 = self.asi8 other_i8 = other.asi8 @@ -779,11 +778,11 @@ def _sub_datetimelike_scalar(self, other): if other is NaT: # type: ignore[comparison-overlap] return self - NaT - if not self._has_same_tz(other): - # require tz compat - raise TypeError( - "Timestamp subtraction must have the same timezones or no timezones" - ) + try: + self._assert_tzawareness_compat(other) + except TypeError as error: + new_message = str(error).replace("compare", "subtract") + raise type(error)(new_message) from error i8 = self.asi8 result = checked_add_with_arr(i8, -other.value, arr_mask=self._isnan) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 8194f47541e4c..6ddf63c03e4f1 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -837,6 +837,61 @@ def test_dt64arr_sub_timedeltalike_scalar( rng -= two_hours tm.assert_equal(rng, expected) + def test_dt64_array_sub_dt_with_different_timezone(self, box_with_array): + t1 = date_range("20130101", periods=3).tz_localize("US/Eastern") + t1 = tm.box_expected(t1, box_with_array) + t2 = Timestamp("20130101").tz_localize("CET") + tnaive = Timestamp(20130101) + + result = t1 - t2 + expected = TimedeltaIndex( + ["0 days 06:00:00", "1 days 06:00:00", "2 days 06:00:00"] + ) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) + + result = t2 - t1 + expected = TimedeltaIndex( + ["-1 days +18:00:00", "-2 days +18:00:00", "-3 days +18:00:00"] + ) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) + + msg = "Cannot subtract tz-naive and tz-aware datetime-like objects" + with pytest.raises(TypeError, match=msg): + t1 - tnaive + + with pytest.raises(TypeError, match=msg): + tnaive - t1 + + def test_dt64_array_sub_dt64_array_with_different_timezone(self, box_with_array): + t1 = date_range("20130101", periods=3).tz_localize("US/Eastern") + t1 = tm.box_expected(t1, box_with_array) + t2 = date_range("20130101", periods=3).tz_localize("CET") + t2 = tm.box_expected(t2, box_with_array) + tnaive = date_range("20130101", periods=3) + + result = t1 - t2 + expected = TimedeltaIndex( + ["0 days 06:00:00", "0 days 06:00:00", "0 days 06:00:00"] + ) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) + + result = t2 - t1 + expected = TimedeltaIndex( + ["-1 days +18:00:00", "-1 days +18:00:00", "-1 days +18:00:00"] + ) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) + + msg = "Cannot subtract tz-naive and tz-aware datetime-like objects" + with pytest.raises(TypeError, match=msg): + t1 - tnaive + + with pytest.raises(TypeError, match=msg): + tnaive - t1 + def test_dt64arr_add_sub_td64_nat(self, box_with_array, tz_naive_fixture): # GH#23320 special handling for timedelta64("NaT") tz = tz_naive_fixture @@ -979,7 +1034,7 @@ def test_dt64arr_aware_sub_dt64ndarray_raises( dt64vals = dti.values dtarr = tm.box_expected(dti, box_with_array) - msg = "subtraction must have the same timezones or" + msg = "Cannot subtract tz-naive and tz-aware datetime" with pytest.raises(TypeError, match=msg): dtarr - dt64vals with pytest.raises(TypeError, match=msg): @@ -2099,7 +2154,6 @@ def test_sub_dti_dti(self): dti = date_range("20130101", periods=3) dti_tz = date_range("20130101", periods=3).tz_localize("US/Eastern") - dti_tz2 = date_range("20130101", periods=3).tz_localize("UTC") expected = TimedeltaIndex([0, 0, 0]) result = dti - dti @@ -2107,16 +2161,13 @@ def test_sub_dti_dti(self): result = dti_tz - dti_tz tm.assert_index_equal(result, expected) - msg = "DatetimeArray subtraction must have the same timezones or" + msg = "Cannot subtract tz-naive and tz-aware datetime-like objects" with pytest.raises(TypeError, match=msg): dti_tz - dti with pytest.raises(TypeError, match=msg): dti - dti_tz - with pytest.raises(TypeError, match=msg): - dti_tz - dti_tz2 - # isub dti -= dti tm.assert_index_equal(dti, expected) diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index 64da7cc968c19..0329d71a472a8 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -387,35 +387,29 @@ def _check(result, expected): _check(result, expected) # tz mismatches - msg = "Timestamp subtraction must have the same timezones or no timezones" + msg = "Cannot subtract tz-naive and tz-aware datetime-like objects." with pytest.raises(TypeError, match=msg): dt_tz - ts msg = "can't subtract offset-naive and offset-aware datetimes" with pytest.raises(TypeError, match=msg): dt_tz - dt - msg = "Timestamp subtraction must have the same timezones or no timezones" - with pytest.raises(TypeError, match=msg): - dt_tz - ts_tz2 msg = "can't subtract offset-naive and offset-aware datetimes" with pytest.raises(TypeError, match=msg): dt - dt_tz - msg = "Timestamp subtraction must have the same timezones or no timezones" + msg = "Cannot subtract tz-naive and tz-aware datetime-like objects." with pytest.raises(TypeError, match=msg): ts - dt_tz with pytest.raises(TypeError, match=msg): ts_tz2 - ts with pytest.raises(TypeError, match=msg): ts_tz2 - dt - with pytest.raises(TypeError, match=msg): - ts_tz - ts_tz2 + msg = "Cannot subtract tz-naive and tz-aware" # with dti with pytest.raises(TypeError, match=msg): dti - ts_tz with pytest.raises(TypeError, match=msg): dti_tz - ts - with pytest.raises(TypeError, match=msg): - dti_tz - ts_tz2 result = dti_tz - dt_tz expected = TimedeltaIndex(["0 days", "1 days", "2 days"]) diff --git a/pandas/tests/scalar/timestamp/test_arithmetic.py b/pandas/tests/scalar/timestamp/test_arithmetic.py index 1a8fd2a8199a2..f7349277bac05 100644 --- a/pandas/tests/scalar/timestamp/test_arithmetic.py +++ b/pandas/tests/scalar/timestamp/test_arithmetic.py @@ -1,6 +1,7 @@ from datetime import ( datetime, timedelta, + timezone, ) import numpy as np @@ -99,7 +100,7 @@ def test_rsub_dtscalars(self, tz_naive_fixture): if tz_naive_fixture is None: assert other.to_datetime64() - ts == td else: - msg = "subtraction must have" + msg = "Cannot subtract tz-naive and tz-aware datetime-like objects" with pytest.raises(TypeError, match=msg): other.to_datetime64() - ts @@ -109,6 +110,37 @@ def test_timestamp_sub_datetime(self): assert (ts - dt).days == 1 assert (dt - ts).days == -1 + def test_subtract_tzaware_datetime(self): + t1 = Timestamp("2020-10-22T22:00:00+00:00") + t2 = datetime(2020, 10, 22, 22, tzinfo=timezone.utc) + + result = t1 - t2 + + assert isinstance(result, Timedelta) + assert result == Timedelta("0 days") + + def test_subtract_timestamp_from_different_timezone(self): + t1 = Timestamp("20130101").tz_localize("US/Eastern") + t2 = Timestamp("20130101").tz_localize("CET") + + result = t1 - t2 + + assert isinstance(result, Timedelta) + assert result == Timedelta("0 days 06:00:00") + + def test_subtracting_involving_datetime_with_different_tz(self): + t1 = datetime(2013, 1, 1, tzinfo=timezone(timedelta(hours=-5))) + t2 = Timestamp("20130101").tz_localize("CET") + + result = t1 - t2 + + assert isinstance(result, Timedelta) + assert result == Timedelta("0 days 06:00:00") + + result = t2 - t1 + assert isinstance(result, Timedelta) + assert result == Timedelta("-1 days +18:00:00") + def test_addition_subtraction_types(self): # Assert on the types resulting from Timestamp +/- various date/time # objects From afaa2ee9eb59d203fb1357660e9a15afd082b59b Mon Sep 17 00:00:00 2001 From: Anton Lodder Date: Tue, 2 Mar 2021 20:55:37 -0500 Subject: [PATCH 2/3] TST: add index type coverage to timedelta subtraction tests (#31793) --- pandas/tests/arithmetic/test_timedelta64.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index 0329d71a472a8..543531889531a 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -357,13 +357,15 @@ def test_subtraction_ops(self): expected = DatetimeIndex(["20121231", NaT, "20121230"], name="foo") tm.assert_index_equal(result, expected) - def test_subtraction_ops_with_tz(self): + def test_subtraction_ops_with_tz(self, box_with_array): # check that dt/dti subtraction ops with tz are validated dti = pd.date_range("20130101", periods=3) + dti = tm.box_expected(dti, box_with_array) ts = Timestamp("20130101") dt = ts.to_pydatetime() dti_tz = pd.date_range("20130101", periods=3).tz_localize("US/Eastern") + dti_tz = tm.box_expected(dti_tz, box_with_array) ts_tz = Timestamp("20130101").tz_localize("US/Eastern") ts_tz2 = Timestamp("20130101").tz_localize("CET") dt_tz = ts_tz.to_pydatetime() @@ -413,19 +415,23 @@ def _check(result, expected): result = dti_tz - dt_tz expected = TimedeltaIndex(["0 days", "1 days", "2 days"]) - tm.assert_index_equal(result, expected) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) result = dt_tz - dti_tz expected = TimedeltaIndex(["0 days", "-1 days", "-2 days"]) - tm.assert_index_equal(result, expected) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) result = dti_tz - ts_tz expected = TimedeltaIndex(["0 days", "1 days", "2 days"]) - tm.assert_index_equal(result, expected) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) result = ts_tz - dti_tz expected = TimedeltaIndex(["0 days", "-1 days", "-2 days"]) - tm.assert_index_equal(result, expected) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) result = td - td expected = Timedelta("0 days") @@ -433,7 +439,8 @@ def _check(result, expected): result = dti_tz - td expected = DatetimeIndex(["20121231", "20130101", "20130102"], tz="US/Eastern") - tm.assert_index_equal(result, expected) + expected = tm.box_expected(expected, box_with_array) + tm.assert_equal(result, expected) def test_dti_tdi_numeric_ops(self): # These are normally union/diff set-like ops From ceb24262b04bdf6ff259771c00584bcb5f9974c4 Mon Sep 17 00:00:00 2001 From: Anton Lodder Date: Thu, 8 Jul 2021 00:01:32 -0400 Subject: [PATCH 3/3] TST: 31739 - add coverage of subtracting datetimes w/differing timezones --- pandas/tests/scalar/timestamp/test_arithmetic.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pandas/tests/scalar/timestamp/test_arithmetic.py b/pandas/tests/scalar/timestamp/test_arithmetic.py index f7349277bac05..e0c18a7055a4e 100644 --- a/pandas/tests/scalar/timestamp/test_arithmetic.py +++ b/pandas/tests/scalar/timestamp/test_arithmetic.py @@ -141,6 +141,16 @@ def test_subtracting_involving_datetime_with_different_tz(self): assert isinstance(result, Timedelta) assert result == Timedelta("-1 days +18:00:00") + def test_subtracting_different_timezones(self, tz_aware_fixture): + t_raw = Timestamp("20130101") + t_UTC = t_raw.tz_localize("UTC") + t_diff = t_UTC.tz_convert(tz_aware_fixture) + Timedelta("0 days 05:00:00") + + result = t_diff - t_UTC + + assert isinstance(result, Timedelta) + assert result == Timedelta("0 days 05:00:00") + def test_addition_subtraction_types(self): # Assert on the types resulting from Timestamp +/- various date/time # objects