From 40be9795cca815dbd8d6f6206e143ff8214d0766 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 27 Jan 2020 23:15:51 -0800 Subject: [PATCH 1/3] BUG: Timedelta components no longer rounded with high precision integers --- doc/source/whatsnew/v1.1.0.rst | 2 +- pandas/_libs/tslibs/timedeltas.pyx | 10 +--------- pandas/tests/scalar/timedelta/test_timedelta.py | 13 +++++++++++++ pandas/tests/scalar/timestamp/test_timestamp.py | 9 +-------- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 920919755dc23..ff88a9b9bac3e 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -109,7 +109,7 @@ Datetimelike Timedelta ^^^^^^^^^ -- +- Bug in constructing a :class:`Timedelta` with a high precision integer that would round the :class:`Timedelta` components (:issue:`31354`) - Timezones diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 9c031baf70a77..ad7cf6ae9307d 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -859,14 +859,6 @@ cdef class _Timedelta(timedelta): """ return self.to_timedelta64() - def total_seconds(self): - """ - Total duration of timedelta in seconds (to microsecond precision). - """ - # GH 31043 - # Microseconds precision to avoid confusing tzinfo.utcoffset - return (self.value - self.value % 1000) / 1e9 - def view(self, dtype): """ Array view compatibility. @@ -1250,7 +1242,7 @@ class Timedelta(_Timedelta): return NaT # make timedelta happy - td_base = _Timedelta.__new__(cls, microseconds=int(value) / 1000) + td_base = _Timedelta.__new__(cls, microseconds=int(value) // 1000) td_base.value = value td_base.is_populated = 0 return td_base diff --git a/pandas/tests/scalar/timedelta/test_timedelta.py b/pandas/tests/scalar/timedelta/test_timedelta.py index e1d965bbb14e9..9cdbeb6ab4845 100644 --- a/pandas/tests/scalar/timedelta/test_timedelta.py +++ b/pandas/tests/scalar/timedelta/test_timedelta.py @@ -821,3 +821,16 @@ def test_resolution_deprecated(self): def test_truthiness(value, expected): # https://github.com/pandas-dev/pandas/issues/21484 assert bool(value) is expected + + +def test_timedelta_attribute_precision(): + # GH 31354 + td = Timedelta(1552211999999999872, unit="ns") + result = td.days * 86400 + result += td.seconds + result *= 1000000 + result += td.microseconds + result *= 1000 + result += td.nanoseconds + expected = td.value + assert result == expected diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index e9c0ee12fc000..51915fdd5235f 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1088,20 +1088,13 @@ def test_constructor_ambigous_dst(): assert result == expected -@pytest.mark.xfail( - LooseVersion(compat._optional._get_version(dateutil)) < LooseVersion("2.7.0"), - reason="dateutil moved to Timedelta.total_seconds() in 2.7.0", -) @pytest.mark.parametrize("epoch", [1552211999999999872, 1552211999999999999]) def test_constructor_before_dst_switch(epoch): # GH 31043 # Make sure that calling Timestamp constructor # on time just before DST switch doesn't lead to # nonexistent time or value change - # Works only with dateutil >= 2.7.0 as dateutil overrid - # pandas.Timedelta.total_seconds with - # datetime.timedelta.total_seconds before - ts = Timestamp(epoch, tz="dateutil/US/Pacific") + ts = Timestamp(epoch, tz="dateutil/America/Los_Angeles") result = ts.tz.dst(ts) expected = timedelta(seconds=0) assert Timestamp(ts).value == epoch From 25a689dda6a2a1a5a40a148c9e57e14a79121c57 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 28 Jan 2020 09:45:28 -0800 Subject: [PATCH 2/3] Simplify NaT.total_seconds --- pandas/_libs/tslibs/nattype.pyx | 10 ++-------- pandas/tests/scalar/timestamp/test_timestamp.py | 1 - 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index 357f183b3a845..9f6f401a1a5f5 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -2,7 +2,7 @@ from cpython.object cimport ( PyObject_RichCompare, Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE) -from cpython.datetime cimport (datetime, +from cpython.datetime cimport (datetime, timedelta, PyDateTime_Check, PyDelta_Check, PyDateTime_IMPORT) @@ -276,13 +276,6 @@ cdef class _NaT(datetime): def __long__(self): return NPY_NAT - def total_seconds(self): - """ - Total duration of timedelta in seconds (to microsecond precision). - """ - # GH#10939 - return np.nan - @property def is_leap_year(self): return False @@ -386,6 +379,7 @@ class NaTType(_NaT): # nan methods weekday = _make_nan_func('weekday', datetime.weekday.__doc__) isoweekday = _make_nan_func('isoweekday', datetime.isoweekday.__doc__) + total_seconds = _make_nan_func('total_seconds', timedelta.total_seconds.__doc__) month_name = _make_nan_func('month_name', # noqa:E128 """ Return the month name of the Timestamp with specified locale. diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 51915fdd5235f..00081018a5bc2 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -2,7 +2,6 @@ import calendar from datetime import datetime, timedelta -from distutils.version import LooseVersion import locale import unicodedata From a4ca7a32ca0fd7d3ffde47e8891600b7c7e18cd8 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 28 Jan 2020 11:10:11 -0800 Subject: [PATCH 3/3] Also simplify another tz conversion code path --- pandas/_libs/tslibs/conversion.pyx | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index e0862b9250045..bf38fcfb6103c 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -29,7 +29,7 @@ from pandas._libs.tslibs.util cimport ( from pandas._libs.tslibs.timedeltas cimport cast_from_unit from pandas._libs.tslibs.timezones cimport ( is_utc, is_tzlocal, is_fixed_offset, get_utcoffset, get_dst_info, - get_timezone, maybe_get_tz, tz_compare, treat_tz_as_dateutil) + get_timezone, maybe_get_tz, tz_compare) from pandas._libs.tslibs.timezones import UTC from pandas._libs.tslibs.parsing import parse_datetime_string @@ -341,14 +341,6 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz, obj.tzinfo = tz else: obj.value = pydatetime_to_dt64(ts, &obj.dts) - # GH 24329 When datetime is ambiguous, - # pydatetime_to_dt64 doesn't take DST into account - # but with dateutil timezone, get_utcoffset does - # so we need to correct for it - if treat_tz_as_dateutil(ts.tzinfo): - if ts.tzinfo.is_ambiguous(ts): - dst_offset = ts.tzinfo.dst(ts) - obj.value += int(dst_offset.total_seconds() * 1e9) obj.tzinfo = ts.tzinfo if obj.tzinfo is not None and not is_utc(obj.tzinfo):