From 2c4ee4d995dace45189751a55cd4e3038f8c7cb6 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 16 Mar 2019 15:44:10 -0400 Subject: [PATCH 1/8] BUG: Fix localize_pydatetime using meta datetimes as Timestamp (#25734) --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/_libs/tslibs/conversion.pyx | 6 +++--- pandas/_libs/tslibs/timestamps.pyx | 2 ++ pandas/_libs/tslibs/util.pxd | 15 +++++++++++++++ pandas/tests/tslibs/test_conversion.py | 26 ++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index ddc5e543c6165..75bbd99ea4dd6 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -167,7 +167,7 @@ Numeric Conversion ^^^^^^^^^^ -- +- Bug in :meth:`localize_pydatetime` where some ``datetime`` were incorrectly treated as pandas :class:`Timestamp` (:issue:`25734`) - - diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 1c0adaaa288a9..3692695c01ee7 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -27,7 +27,8 @@ from pandas._libs.tslibs.np_datetime cimport ( from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime from pandas._libs.tslibs.util cimport ( - is_string_object, is_datetime64_object, is_integer_object, is_float_object) + is_string_object, is_datetime64_object, is_integer_object, is_float_object, + is_timestamp) from pandas._libs.tslibs.timedeltas cimport (cast_from_unit, delta_to_nanoseconds) @@ -607,8 +608,7 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz): """ if tz is None: return dt - elif not PyDateTime_CheckExact(dt): - # i.e. is a Timestamp + elif is_timestamp(dt): return dt.tz_localize(tz) elif is_utc(tz): return _localize_pydatetime(dt, tz) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 8d825e0a6179e..e9668df9abce0 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -203,6 +203,8 @@ def round_nsint64(values, mode, freq): # shadows the python class, where we do any heavy lifting. cdef class _Timestamp(datetime): + _typ = 'timestamp' + cdef readonly: int64_t value, nanosecond object freq # frequency reference diff --git a/pandas/_libs/tslibs/util.pxd b/pandas/_libs/tslibs/util.pxd index ef7065a44f18b..8202db93f3fd8 100644 --- a/pandas/_libs/tslibs/util.pxd +++ b/pandas/_libs/tslibs/util.pxd @@ -227,3 +227,18 @@ cdef inline bint is_nan(object val): is_nan : bool """ return (is_float_object(val) or is_complex_object(val)) and val != val + + +cdef inline bint is_timestamp(object val): + """ + Cython equivalent of `isinstance(val, pd.Timestamp)` + + Parameters + ---------- + val : object + + Returns + ------- + is_timestamp : bool + """ + return getattr(val, '_typ', None) == "timestamp" diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index 13398a69b4982..91ae9e58e1bb8 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -3,6 +3,7 @@ import numpy as np import pytest from pytz import UTC +from datetime import datetime from pandas._libs.tslib import iNaT from pandas._libs.tslibs import conversion, timezones @@ -10,6 +11,8 @@ from pandas import date_range import pandas.util.testing as tm +from pandas import Timestamp + def _compare_utc_to_local(tz_didx): def f(x): @@ -66,3 +69,26 @@ def test_length_zero_copy(dtype, copy): arr = np.array([], dtype=dtype) result = conversion.ensure_datetime64ns(arr, copy=copy) assert result.base is (None if copy else arr) + + +class MetaDatetime(type): + pass + + +class FakeDatetime(MetaDatetime("NewBase", (datetime,), {})): + pass + + +@pytest.mark.parametrize("dt, expected", [ + pytest.param(Timestamp("2000-01-01"), + Timestamp("2000-01-01", tz=UTC), id="timestamp"), + pytest.param(datetime(2000, 1, 1), + datetime(2000, 1, 1, tzinfo=UTC), + id="datetime"), + pytest.param(FakeDatetime(2000, 1, 1), + FakeDatetime(2000, 1, 1, tzinfo=UTC), + id="fakedatetime")]) +def test_localize_pydatetime_dt_types(dt, expected): + # GH 25734 + result = conversion.localize_pydatetime(dt, UTC) + assert result == expected From de72ec6318173415fe264d8539a9d7205aa09d36 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sun, 17 Mar 2019 14:33:32 -0400 Subject: [PATCH 2/8] PR fixes, additional fix in convert_datetime_to_tsobject --- doc/source/whatsnew/v0.25.0.rst | 4 +-- pandas/_libs/tslibs/conversion.pyx | 18 +++++++----- pandas/_libs/tslibs/timestamps.pyx | 2 -- pandas/_libs/tslibs/util.pxd | 15 ---------- pandas/tests/arithmetic/test_datetime64.py | 33 ++++++++++++++++++++++ pandas/tests/tslibs/test_conversion.py | 10 ++----- 6 files changed, 48 insertions(+), 34 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 75bbd99ea4dd6..1e65f0a103e0c 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -136,7 +136,7 @@ Categorical Datetimelike ^^^^^^^^^^^^ -- +- Bug with :class:`Timestamp` and :class:`CustomBusinessDay` arithmatic throwing an exception with datetime subclasses (:issue:`25734`) - - @@ -167,7 +167,7 @@ Numeric Conversion ^^^^^^^^^^ -- Bug in :meth:`localize_pydatetime` where some ``datetime`` were incorrectly treated as pandas :class:`Timestamp` (:issue:`25734`) +- - - diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 3692695c01ee7..0cf45775daad0 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -27,8 +27,7 @@ from pandas._libs.tslibs.np_datetime cimport ( from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime from pandas._libs.tslibs.util cimport ( - is_string_object, is_datetime64_object, is_integer_object, is_float_object, - is_timestamp) + is_string_object, is_datetime64_object, is_integer_object, is_float_object) from pandas._libs.tslibs.timedeltas cimport (cast_from_unit, delta_to_nanoseconds) @@ -381,9 +380,11 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz, obj.value -= int(offset.total_seconds() * 1e9) if not PyDateTime_CheckExact(ts): - # datetime instance but not datetime type --> Timestamp - obj.value += ts.nanosecond - obj.dts.ps = ts.nanosecond * 1000 + try: + obj.value += ts.nanosecond + obj.dts.ps = ts.nanosecond * 1000 + except AttributeError: + pass if nanos: obj.value += nanos @@ -608,8 +609,11 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz): """ if tz is None: return dt - elif is_timestamp(dt): - return dt.tz_localize(tz) + elif not PyDateTime_CheckExact(dt): + try: + return dt.tz_localize(tz) + except AttributeError: + pass elif is_utc(tz): return _localize_pydatetime(dt, tz) try: diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index e9668df9abce0..8d825e0a6179e 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -203,8 +203,6 @@ def round_nsint64(values, mode, freq): # shadows the python class, where we do any heavy lifting. cdef class _Timestamp(datetime): - _typ = 'timestamp' - cdef readonly: int64_t value, nanosecond object freq # frequency reference diff --git a/pandas/_libs/tslibs/util.pxd b/pandas/_libs/tslibs/util.pxd index 8202db93f3fd8..ef7065a44f18b 100644 --- a/pandas/_libs/tslibs/util.pxd +++ b/pandas/_libs/tslibs/util.pxd @@ -227,18 +227,3 @@ cdef inline bint is_nan(object val): is_nan : bool """ return (is_float_object(val) or is_complex_object(val)) and val != val - - -cdef inline bint is_timestamp(object val): - """ - Cython equivalent of `isinstance(val, pd.Timestamp)` - - Parameters - ---------- - val : object - - Returns - ------- - is_timestamp : bool - """ - return getattr(val, '_typ', None) == "timestamp" diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index c81a371f37dc1..76f5a1bb7ffeb 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta from itertools import product, starmap import operator +import sys import warnings import numpy as np @@ -15,6 +16,7 @@ from pandas._libs.tslibs.offsets import shift_months from pandas.compat.numpy import np_datetime64_compat from pandas.errors import NullFrequencyError, PerformanceWarning +from pandas.tseries.offsets import CustomBusinessDay import pandas as pd from pandas import ( @@ -2350,3 +2352,34 @@ def test_shift_months(years, months): for x in dti] expected = DatetimeIndex(raw) tm.assert_index_equal(actual, expected) + + +def test_add_with_monkeypatched_datetime(monkeypatch): + # GH 25734 + + class MetaDatetime(type): + @classmethod + def __instancecheck__(self, obj): + return isinstance(obj, datetime) + + class FakeDatetime(MetaDatetime("NewBase", (datetime,), {})): + pass + + with monkeypatch.context() as m: + # monkeypatch datetime everywhere + for mod_name, module in list(sys.modules.items()): + if (mod_name == __name__ or + module.__name__ in ('datetime',)): + continue + for attribute_name in dir(module): + try: + attribute_value = getattr(module, attribute_name) + except (ImportError, AttributeError, TypeError): + continue + if id(datetime) == id(attribute_value): + m.setattr(module, attribute_name, FakeDatetime) + + dt = FakeDatetime(2000, 1, 1, tzinfo=pytz.UTC) + result = Timestamp(dt) + CustomBusinessDay() + expected = Timestamp("2000-01-03", tzinfo=pytz.UTC) + assert result == expected diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index 91ae9e58e1bb8..b202a4c150047 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -8,11 +8,9 @@ from pandas._libs.tslib import iNaT from pandas._libs.tslibs import conversion, timezones -from pandas import date_range +from pandas import date_range, Timestamp import pandas.util.testing as tm -from pandas import Timestamp - def _compare_utc_to_local(tz_didx): def f(x): @@ -71,11 +69,7 @@ def test_length_zero_copy(dtype, copy): assert result.base is (None if copy else arr) -class MetaDatetime(type): - pass - - -class FakeDatetime(MetaDatetime("NewBase", (datetime,), {})): +class FakeDatetime(datetime): pass From f585ecd0abfd1598b863023a0e390c9d440529d6 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sun, 17 Mar 2019 15:08:51 -0400 Subject: [PATCH 3/8] fix isort errors --- pandas/tests/arithmetic/test_datetime64.py | 2 +- pandas/tests/tslibs/test_conversion.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 76f5a1bb7ffeb..1894a7e4528f5 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -16,13 +16,13 @@ from pandas._libs.tslibs.offsets import shift_months from pandas.compat.numpy import np_datetime64_compat from pandas.errors import NullFrequencyError, PerformanceWarning -from pandas.tseries.offsets import CustomBusinessDay import pandas as pd from pandas import ( DatetimeIndex, NaT, Period, Series, Timedelta, TimedeltaIndex, Timestamp, date_range) from pandas.core.indexes.datetimes import _to_M8 +from pandas.tseries.offsets import CustomBusinessDay import pandas.util.testing as tm diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index b202a4c150047..7b9b2371b6926 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- +from datetime import datetime + import numpy as np import pytest from pytz import UTC -from datetime import datetime from pandas._libs.tslib import iNaT from pandas._libs.tslibs import conversion, timezones From 194caf585c4cb5c2fcccd67c1ff44762816a048a Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Sun, 17 Mar 2019 20:07:16 +0000 Subject: [PATCH 4/8] fix isort errors properly --- pandas/tests/arithmetic/test_datetime64.py | 3 ++- pandas/tests/tslibs/test_conversion.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 1894a7e4528f5..d2bd7c67676d9 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -22,9 +22,10 @@ DatetimeIndex, NaT, Period, Series, Timedelta, TimedeltaIndex, Timestamp, date_range) from pandas.core.indexes.datetimes import _to_M8 -from pandas.tseries.offsets import CustomBusinessDay import pandas.util.testing as tm +from pandas.tseries.offsets import CustomBusinessDay + def assert_all(obj): """ diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index 7b9b2371b6926..4b4345da18d1c 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -9,7 +9,7 @@ from pandas._libs.tslib import iNaT from pandas._libs.tslibs import conversion, timezones -from pandas import date_range, Timestamp +from pandas import Timestamp, date_range import pandas.util.testing as tm From 2de4d15447bcd21c716357c68fa28c29e57473d8 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sun, 17 Mar 2019 21:19:38 -0400 Subject: [PATCH 5/8] typo, comments, improve test exception handling --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/_libs/tslibs/conversion.pyx | 2 ++ pandas/tests/arithmetic/test_datetime64.py | 7 +++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 1e65f0a103e0c..8dbfa2a898694 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -136,7 +136,7 @@ Categorical Datetimelike ^^^^^^^^^^^^ -- Bug with :class:`Timestamp` and :class:`CustomBusinessDay` arithmatic throwing an exception with datetime subclasses (:issue:`25734`) +- Bug with :class:`Timestamp` and :class:`CustomBusinessDay` arithmetic throwing an exception with datetime subclasses (:issue:`25734`) - - diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 0cf45775daad0..5c1ea95676897 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -384,6 +384,7 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz, obj.value += ts.nanosecond obj.dts.ps = ts.nanosecond * 1000 except AttributeError: + # probably a subclass of datetime pass if nanos: @@ -613,6 +614,7 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz): try: return dt.tz_localize(tz) except AttributeError: + # probably a subclass of datetime pass elif is_utc(tz): return _localize_pydatetime(dt, tz) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index d2bd7c67676d9..e62846f5afa2e 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2369,8 +2369,11 @@ class FakeDatetime(MetaDatetime("NewBase", (datetime,), {})): with monkeypatch.context() as m: # monkeypatch datetime everywhere for mod_name, module in list(sys.modules.items()): - if (mod_name == __name__ or - module.__name__ in ('datetime',)): + try: + if (mod_name == __name__ or + module.__name__ in ('datetime',)): + continue + except AttributeError: continue for attribute_name in dir(module): try: From 849493c6c0b68fe57784a5811158325c6fd81d48 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sun, 17 Mar 2019 21:20:58 -0400 Subject: [PATCH 6/8] indentation fix --- pandas/tests/arithmetic/test_datetime64.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index e62846f5afa2e..2b10a0d17f3ac 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2371,7 +2371,7 @@ class FakeDatetime(MetaDatetime("NewBase", (datetime,), {})): for mod_name, module in list(sys.modules.items()): try: if (mod_name == __name__ or - module.__name__ in ('datetime',)): + module.__name__ in ('datetime',)): continue except AttributeError: continue From c7734075d1f09198b445368122c83eacdacb6bdb Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sun, 17 Mar 2019 21:39:52 -0400 Subject: [PATCH 7/8] improve test exception handling --- pandas/tests/arithmetic/test_datetime64.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 2b10a0d17f3ac..c4e976c1d9c20 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2373,7 +2373,7 @@ class FakeDatetime(MetaDatetime("NewBase", (datetime,), {})): if (mod_name == __name__ or module.__name__ in ('datetime',)): continue - except AttributeError: + except (ImportError, AttributeError, TypeError): continue for attribute_name in dir(module): try: From 27657f1d7bca751dd734f16d68b3e82b9cac7f9a Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sun, 17 Mar 2019 22:37:46 -0400 Subject: [PATCH 8/8] even better test exception handling --- pandas/tests/arithmetic/test_datetime64.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index c4e976c1d9c20..4ee635025fbe1 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2373,9 +2373,10 @@ class FakeDatetime(MetaDatetime("NewBase", (datetime,), {})): if (mod_name == __name__ or module.__name__ in ('datetime',)): continue + module_attributes = dir(module) except (ImportError, AttributeError, TypeError): continue - for attribute_name in dir(module): + for attribute_name in module_attributes: try: attribute_value = getattr(module, attribute_name) except (ImportError, AttributeError, TypeError):