BUG: Fix Timestamp type checks to work with subclassed datetime (#25851)

doc/source/whatsnew/v0.25.0.rst

@@ -325,7 +325,7 @@ Sparse
 Other
 ^^^^^
 
--
+- Improved :class:`Timestamp` type checking in various datetime functions to prevent exceptions when using a subclassed `datetime` (:issue:`25851`)
 -
 -
 

pandas/_libs/tslib.pyx

@@ -2,7 +2,6 @@
 import cython
 
 from cpython.datetime cimport (PyDateTime_Check, PyDate_Check,
-                               PyDateTime_CheckExact,
                                PyDateTime_IMPORT,
                                timedelta, datetime, date, time)
 # import datetime C API
@@ -41,7 +40,8 @@ from pandas._libs.tslibs.nattype cimport (
 
 from pandas._libs.tslibs.offsets cimport to_offset
 
-from pandas._libs.tslibs.timestamps cimport create_timestamp_from_ts
+from pandas._libs.tslibs.timestamps cimport (
+    create_timestamp_from_ts, _Timestamp)
 from pandas._libs.tslibs.timestamps import Timestamp
 
 
@@ -539,8 +539,7 @@ cpdef array_to_datetime(ndarray[object] values, str errors='raise',
                                              'datetime64 unless utc=True')
                     else:
                         iresult[i] = pydatetime_to_dt64(val, &dts)
-                        if not PyDateTime_CheckExact(val):
-                            # i.e. a Timestamp object
+                        if isinstance(val, _Timestamp):
                             iresult[i] += val.nanosecond
                         check_dts_bounds(&dts)
 

pandas/_libs/tslibs/__init__.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
 
+from .timestamps import Timestamp  # isort:skip
 from .conversion import normalize_date, localize_pydatetime, tz_convert_single
 from .nattype import NaT, NaTType, iNaT, is_null_datetimelike
 from .np_datetime import OutOfBoundsDatetime
 from .period import Period, IncompatibleFrequency
-from .timestamps import Timestamp
 from .timedeltas import delta_to_nanoseconds, ints_to_pytimedelta, Timedelta

pandas/_libs/tslibs/conversion.pyx

@@ -13,8 +13,7 @@ from dateutil.tz import tzutc
 from datetime import time as datetime_time
 from cpython.datetime cimport (datetime, tzinfo,
                                PyDateTime_Check, PyDate_Check,
-                               PyDateTime_CheckExact, PyDateTime_IMPORT,
-                               PyDelta_Check)
+                               PyDateTime_IMPORT, PyDelta_Check)
 PyDateTime_IMPORT
 
 from pandas._libs.tslibs.ccalendar import DAY_SECONDS, HOUR_SECONDS
@@ -31,6 +30,7 @@ from pandas._libs.tslibs.util cimport (
 
 from pandas._libs.tslibs.timedeltas cimport (cast_from_unit,
                                              delta_to_nanoseconds)
+from pandas._libs.tslibs.timestamps cimport _Timestamp
 from pandas._libs.tslibs.timezones cimport (
     is_utc, is_tzlocal, is_fixed_offset, get_utcoffset, get_dst_info,
     get_timezone, maybe_get_tz, tz_compare)
@@ -379,8 +379,7 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz,
         offset = get_utcoffset(obj.tzinfo, ts)
         obj.value -= int(offset.total_seconds() * 1e9)
 
-    if not PyDateTime_CheckExact(ts):
-        # datetime instance but not datetime type --> Timestamp
+    if isinstance(ts, _Timestamp):
         obj.value += ts.nanosecond
         obj.dts.ps = ts.nanosecond * 1000
 
@@ -607,8 +606,7 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz):
     """
     if tz is None:
         return dt
-    elif not PyDateTime_CheckExact(dt):
-        # i.e. is a Timestamp
+    elif isinstance(dt, _Timestamp):
         return dt.tz_localize(tz)
     elif is_utc(tz):
         return _localize_pydatetime(dt, tz)
@@ -1155,8 +1153,7 @@ def normalize_date(dt: object) -> datetime:
     TypeError : if input is not datetime.date, datetime.datetime, or Timestamp
     """
     if PyDateTime_Check(dt):
-        if not PyDateTime_CheckExact(dt):
-            # i.e. a Timestamp object
+        if isinstance(dt, _Timestamp):
             return dt.replace(hour=0, minute=0, second=0, microsecond=0,
                               nanosecond=0)
         else:

pandas/_libs/tslibs/timedeltas.pyx

@@ -16,7 +16,6 @@ from numpy cimport int64_t
 cnp.import_array()
 
 from cpython.datetime cimport (datetime, timedelta,
-                               PyDateTime_CheckExact,
                                PyDateTime_Check, PyDelta_Check,
                                PyDateTime_IMPORT)
 PyDateTime_IMPORT
@@ -37,6 +36,7 @@ from pandas._libs.tslibs.nattype cimport (
     checknull_with_nat, NPY_NAT, c_NaT as NaT)
 from pandas._libs.tslibs.offsets cimport to_offset
 from pandas._libs.tslibs.offsets import _Tick as Tick
+from pandas._libs.tslibs.timestamps cimport _Timestamp
 
 # ----------------------------------------------------------------------
 # Constants
@@ -583,9 +583,10 @@ def _binary_op_method_timedeltalike(op, name):
             # has-dtype check before then
             pass
 
-        elif is_datetime64_object(other) or PyDateTime_CheckExact(other):
-            # the PyDateTime_CheckExact case is for a datetime object that
-            # is specifically *not* a Timestamp, as the Timestamp case will be
+        elif is_datetime64_object(other) or (
+           PyDateTime_Check(other) and not isinstance(other, _Timestamp)):
+            # this case is for a datetime object that is specifically
+            # *not* a Timestamp, as the Timestamp case will be
             # handled after `_validate_ops_compat` returns False below
             from pandas._libs.tslibs.timestamps import Timestamp
             return op(self, Timestamp(other))

pandas/_libs/tslibs/timestamps.pxd

@@ -1,8 +1,24 @@
 # -*- coding: utf-8 -*-
 
+from cpython.datetime cimport datetime
+
 from numpy cimport int64_t
 from pandas._libs.tslibs.np_datetime cimport npy_datetimestruct
 
 cdef object create_timestamp_from_ts(int64_t value,
                                      npy_datetimestruct dts,
                                      object tz, object freq)
+
+cdef class _Timestamp(datetime):
+    cdef readonly:
+        int64_t value, nanosecond
+        object freq
+        list _date_attributes
+    cpdef bint _get_start_end_field(self, str field)
+    cpdef _get_date_name_field(self, object field, object locale)
+    cdef int64_t _maybe_convert_value_to_local(self)
+    cpdef to_datetime64(self)
+    cdef _assert_tzawareness_compat(_Timestamp self, datetime other)
+    cpdef datetime to_pydatetime(_Timestamp self, bint warn=*)
+    cdef bint _compare_outside_nanorange(_Timestamp self, datetime other,
+                                         int op) except -1

pandas/_libs/tslibs/timestamps.pyx

@@ -22,8 +22,6 @@ from pandas._libs.tslibs.util cimport (
 
 cimport pandas._libs.tslibs.ccalendar as ccalendar
 from pandas._libs.tslibs.ccalendar import DAY_SECONDS
-from pandas._libs.tslibs.conversion import (
-    tz_localize_to_utc, normalize_i8_timestamps)
 from pandas._libs.tslibs.conversion cimport (
     tz_convert_single, _TSObject, convert_to_tsobject,
     convert_datetime_to_tsobject)
@@ -203,11 +201,6 @@ def round_nsint64(values, mode, freq):
 # shadows the python class, where we do any heavy lifting.
 cdef class _Timestamp(datetime):
 
-    cdef readonly:
-        int64_t value, nanosecond
-        object freq       # frequency reference
-        list _date_attributes
-
     def __hash__(_Timestamp self):
         if self.nanosecond:
             return hash(self.value)
@@ -1215,6 +1208,7 @@ class Timestamp(_Timestamp):
             tz = maybe_get_tz(tz)
             if not is_string_object(ambiguous):
                 ambiguous = [ambiguous]
+            from pandas._libs.tslibs.conversion import tz_localize_to_utc
             value = tz_localize_to_utc(np.array([self.value], dtype='i8'), tz,
                                        ambiguous=ambiguous,
                                        nonexistent=nonexistent)[0]
@@ -1409,6 +1403,7 @@ class Timestamp(_Timestamp):
             DAY_NS = DAY_SECONDS * 1000000000
             normalized_value = self.value - (self.value % DAY_NS)
             return Timestamp(normalized_value).tz_localize(self.tz)
+        from pandas._libs.tslibs.conversion import normalize_i8_timestamps
         normalized_value = normalize_i8_timestamps(
             np.array([self.value], dtype='i8'), tz=self.tz)[0]
         return Timestamp(normalized_value).tz_localize(self.tz)

pandas/tests/arithmetic/test_datetime64.py

@@ -2351,3 +2351,14 @@ def test_shift_months(years, months):
            for x in dti]
     expected = DatetimeIndex(raw)
     tm.assert_index_equal(actual, expected)
+
+
+def test_dt_subclass_add_timedelta():
+    # GH 25851
+    class SubDatetime(datetime):
+        pass
+    dt = SubDatetime(2000, 1, 1)
+    td = Timedelta(hours=1)
+    result = dt + td
+    expected = SubDatetime(2000, 1, 1, 1)
+    assert result == expected

pandas/tests/scalar/timestamp/test_timestamp.py

@@ -604,6 +604,15 @@ def test_dont_convert_dateutil_utc_to_pytz_utc(self):
         expected = Timestamp(datetime(2018, 1, 1)).tz_localize(tzutc())
         assert result == expected
 
+    def test_constructor_subclassed_datetime(self):
+        # GH 25851
+        class SubDatetime(datetime):
+            pass
+        data = SubDatetime(2000, 1, 1)
+        result = Timestamp(data)
+        expected = Timestamp(2000, 1, 1)
+        assert result == expected
+
 
 class TestTimestamp(object):
 

pandas/tests/tslibs/test_array_to_datetime.py

@@ -9,6 +9,7 @@
 from pandas._libs import iNaT, tslib
 from pandas.compat.numpy import np_array_datetime64_compat
 
+from pandas import Timestamp
 import pandas.util.testing as tm
 
 
@@ -154,3 +155,25 @@ def test_to_datetime_barely_out_of_bounds():
 
     with pytest.raises(tslib.OutOfBoundsDatetime, match=msg):
         tslib.array_to_datetime(arr)
+
+
+class SubDatetime(datetime):
+    pass
+
+
+@pytest.mark.parametrize("data,expected", [
+    ([SubDatetime(2000, 1, 1)],
+     ["2000-01-01T00:00:00.000000000-0000"]),
+    ([datetime(2000, 1, 1)],
+     ["2000-01-01T00:00:00.000000000-0000"]),
+    ([Timestamp(2000, 1, 1)],
+     ["2000-01-01T00:00:00.000000000-0000"])
+])
+def test_datetime_subclass(data, expected):
+    # GH 25851
+
+    arr = np.array(data, dtype=object)
+    result, _ = tslib.array_to_datetime(arr)
+
+    expected = np_array_datetime64_compat(expected, dtype="M8[ns]")
+    tm.assert_numpy_array_equal(result, expected)

pandas/tests/tslibs/test_conversion.py

@@ -1,13 +1,15 @@
 # -*- coding: utf-8 -*-
 
+from datetime import datetime
+
 import numpy as np
 import pytest
 from pytz import UTC
 
 from pandas._libs.tslib import iNaT
 from pandas._libs.tslibs import conversion, timezones
 
-from pandas import date_range
+from pandas import Timestamp, date_range
 import pandas.util.testing as tm
 
 
@@ -66,3 +68,22 @@ def test_length_zero_copy(dtype, copy):
     arr = np.array([], dtype=dtype)
     result = conversion.ensure_datetime64ns(arr, copy=copy)
     assert result.base is (None if copy else arr)
+
+
+class SubDatetime(datetime):
+    pass
+
+
+@pytest.mark.parametrize("dt, expected", [
+    pytest.param(Timestamp("2000-01-01"),
+                 Timestamp("2000-01-01", tz=UTC), id="timestamp"),
+    pytest.param(datetime(2000, 1, 1),
+                 datetime(2000, 1, 1, tzinfo=UTC),
+                 id="datetime"),
+    pytest.param(SubDatetime(2000, 1, 1),
+                 SubDatetime(2000, 1, 1, tzinfo=UTC),
+                 id="subdatetime")])
+def test_localize_pydatetime_dt_types(dt, expected):
+    # GH 25851
+    result = conversion.localize_pydatetime(dt, UTC)
+    assert result == expected

pandas/tests/tslibs/test_normalize_date.py

@@ -7,6 +7,8 @@
 
 from pandas._libs import tslibs
 
+from pandas import Timestamp
+
 
 @pytest.mark.parametrize("value,expected", [
     (date(2012, 9, 7), datetime(2012, 9, 7)),
@@ -16,3 +18,20 @@
 def test_normalize_date(value, expected):
     result = tslibs.normalize_date(value)
     assert result == expected
+
+
+class SubDatetime(datetime):
+    pass
+
+
+@pytest.mark.parametrize("dt, expected", [
+    pytest.param(Timestamp(2000, 1, 1, 1),
+                 Timestamp(2000, 1, 1, 0)),
+    pytest.param(datetime(2000, 1, 1, 1),
+                 datetime(2000, 1, 1, 0)),
+    pytest.param(SubDatetime(2000, 1, 1, 1),
+                 SubDatetime(2000, 1, 1, 0))])
+def test_normalize_date_sub_types(dt, expected):
+    # GH 25851
+    result = tslibs.normalize_date(dt)
+    assert result == expected