BUG: hashing datetime64 objects

pandas/_libs/src/klib/khash_python.h

@@ -7,7 +7,10 @@
 typedef npy_complex64 khcomplex64_t;
 typedef npy_complex128 khcomplex128_t;
 
+// get pandas_datetime_to_datetimestruct
+#include <../../tslibs/src/datetime/np_datetime.h>
 
+#include "datetime.h"
 
 // khash should report usage to tracemalloc
 #if PY_VERSION_HEX >= 0x03060000
@@ -305,6 +308,7 @@ khuint32_t PANDAS_INLINE kh_python_hash_func(PyObject* key);
 #define _PandasHASH_XXROTATE(x) ((x << 13) | (x >> 19))  /* Rotate left 13 bits */
 #endif
 
+
 Py_hash_t PANDAS_INLINE tupleobject_hash(PyTupleObject* key) {
     Py_ssize_t i, len = Py_SIZE(key);
     PyObject **item = key->ob_item;
@@ -315,9 +319,7 @@ Py_hash_t PANDAS_INLINE tupleobject_hash(PyTupleObject* key) {
         if (lane == (Py_uhash_t)-1) {
             return -1;
         }
-        acc += lane * _PandasHASH_XXPRIME_2;
-        acc = _PandasHASH_XXROTATE(acc);
-        acc *= _PandasHASH_XXPRIME_1;
+        acc = tuple_update_uhash(acc, lane);
     }
 
     /* Add input length, mangled to keep the historical value of hash(()). */
@@ -330,6 +332,46 @@ Py_hash_t PANDAS_INLINE tupleobject_hash(PyTupleObject* key) {
 }
 
 
+// TODO: same thing for timedelta64 objects
+Py_hash_t np_datetime64_object_hash(PyDatetimeScalarObject* key) {
+    // GH#50690 numpy's hash implementation does not preserve comparabity
+    // either across resolutions or with standard library objects.
+    // See also Timestamp.__hash__
+
+    NPY_DATETIMEUNIT unit = (NPY_DATETIMEUNIT)key->obmeta.base;
+    npy_datetime value = key->obval;
+    npy_datetimestruct dts;
+
+    if (value == NPY_DATETIME_NAT) {
+        // np.datetime64("NaT") in any reso
+        return NPY_DATETIME_NAT;
+    }
+
+    pandas_datetime_to_datetimestruct(value, unit, &dts);
+
+    if ((dts.year > 0) && (dts.year <= 9999) && (dts.ps == 0) && (dts.as == 0)) {
+        // we CAN cast to pydatetime, so use that hash to ensure we compare
+        // as matching standard library datetimes (and pd.Timestamps)
+        PyDateTime_IMPORT;
+
+        PyObject* dt;
+        Py_hash_t hash;
+
+        dt = PyDateTime_FromDateAndTime(
+            dts.year, dts.month, dts.day, dts.hour, dts.min, dts.sec, dts.us
+        );
+        if (dt == NULL) {
+            return -1;
+        }
+        hash = PyObject_Hash(dt);
+        Py_DECREF(dt);
+        return hash;
+    }
+
+    return hash_datetime_from_struct(&dts);
+}
+
+
 khuint32_t PANDAS_INLINE kh_python_hash_func(PyObject* key) {
     Py_hash_t hash;
     // For PyObject_Hash holds:
@@ -351,6 +393,10 @@ khuint32_t PANDAS_INLINE kh_python_hash_func(PyObject* key) {
     else if (PyTuple_CheckExact(key)) {
         hash = tupleobject_hash((PyTupleObject*)key);
     }
+    else if (PyObject_TypeCheck(key, &PyDatetimeArrType_Type)) {
+        // GH#50690
+        hash = np_datetime64_object_hash((PyDatetimeScalarObject *)key);
+    }
     else {
         hash = PyObject_Hash(key);
     }

pandas/_libs/tslibs/np_datetime.pxd

@@ -71,6 +71,8 @@ cdef extern from "src/datetime/np_datetime.h":
                                              pandas_timedeltastruct *result
                                              ) nogil
 
+    Py_hash_t hash_datetime_from_struct(npy_datetimestruct* dts) except? -1
+
 cdef bint cmp_scalar(int64_t lhs, int64_t rhs, int op) except -1
 
 cdef check_dts_bounds(npy_datetimestruct *dts, NPY_DATETIMEUNIT unit=?)

pandas/_libs/tslibs/src/datetime/np_datetime.c

@@ -1091,3 +1091,56 @@ PyArray_DatetimeMetaData
 get_datetime_metadata_from_dtype(PyArray_Descr *dtype) {
     return (((PyArray_DatetimeDTypeMetaData *)dtype->c_metadata)->meta);
 }
+
+
+// we could use any hashing algorithm, this is the original CPython's for tuples
+
+#if SIZEOF_PY_UHASH_T > 4
+#define _PandasHASH_XXPRIME_1 ((Py_uhash_t)11400714785074694791ULL)
+#define _PandasHASH_XXPRIME_2 ((Py_uhash_t)14029467366897019727ULL)
+#define _PandasHASH_XXPRIME_5 ((Py_uhash_t)2870177450012600261ULL)
+#define _PandasHASH_XXROTATE(x) ((x << 31) | (x >> 33))  /* Rotate left 31 bits */
+#else
+#define _PandasHASH_XXPRIME_1 ((Py_uhash_t)2654435761UL)
+#define _PandasHASH_XXPRIME_2 ((Py_uhash_t)2246822519UL)
+#define _PandasHASH_XXPRIME_5 ((Py_uhash_t)374761393UL)
+#define _PandasHASH_XXROTATE(x) ((x << 13) | (x >> 19))  /* Rotate left 13 bits */
+#endif
+
+
+Py_uhash_t tuple_update_uhash(Py_uhash_t acc, Py_uhash_t lane) {
+    acc += lane * _PandasHASH_XXPRIME_2;
+    acc = _PandasHASH_XXROTATE(acc);
+    acc *= _PandasHASH_XXPRIME_1;
+    return acc;
+}
+
+// https://github.com/pandas-dev/pandas/pull/50960
+Py_hash_t
+hash_datetime_from_struct(npy_datetimestruct* dts) {
+    /*
+     * If we cannot cast to datetime, use the datetime struct values directly
+     * and mix them similar to a tuple.
+     */
+
+    Py_uhash_t acc = _PandasHASH_XXPRIME_5;
+#if 64 <= SIZEOF_PY_UHASH_T
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->year);
+#else
+    /* Mix lower and uper bits of the year if int64 is larger */
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->year);
+    acc = tuple_update_uhash(acc, (Py_uhash_t)(dts->year >> SIZEOF_PY_UHASH_T));
+#endif
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->month);
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->day);
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->min);
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->sec);
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->us);
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->ps);
+    acc = tuple_update_uhash(acc, (Py_uhash_t)dts->as);
+    /* should be a need to mix length, as it is fixed anyway? */
+    if (acc == (Py_uhash_t)-1) {
+        acc = (Py_uhash_t)-2;
+    }
+    return acc;
+}

pandas/_libs/tslibs/src/datetime/np_datetime.h

@@ -99,4 +99,7 @@ PyArray_DatetimeMetaData get_datetime_metadata_from_dtype(
         PyArray_Descr *dtype);
 
 
+Py_hash_t hash_datetime_from_struct(npy_datetimestruct* dts);
+Py_uhash_t tuple_update_uhash(Py_uhash_t acc, Py_uhash_t lane);
+
 #endif  // PANDAS__LIBS_TSLIBS_SRC_DATETIME_NP_DATETIME_H_

pandas/_libs/tslibs/timestamps.pyx

@@ -88,6 +88,7 @@ from pandas._libs.tslibs.np_datetime cimport (
     get_datetime64_unit,
     get_datetime64_value,
     get_unit_from_dtype,
+    hash_datetime_from_struct,
     npy_datetimestruct,
     npy_datetimestruct_to_datetime,
     pandas_datetime_to_datetimestruct,
@@ -308,11 +309,12 @@ cdef class _Timestamp(ABCTimestamp):
     # -----------------------------------------------------------------
 
     def __hash__(_Timestamp self):
-        if self.nanosecond:
-            return hash(self._value)
-        if not (1 <= self.year <= 9999):
+        cdef:
+            npy_datetimestruct dts
+        if not (1 <= self.year <= 9999) or self.nanosecond:
             # out of bounds for pydatetime
-            return hash(self._value)
+            pydatetime_to_dtstruct(self, &dts)
+            return hash_datetime_from_struct(&dts)
         if self.fold:
             return datetime.__hash__(self.replace(fold=0))
         return datetime.__hash__(self)

pandas/tests/indexes/object/test_indexing.py

@@ -187,3 +187,29 @@ def test_slice_locs_dup(self):
         assert index2.slice_locs(end="a") == (0, 6)
         assert index2.slice_locs("d", "b") == (0, 4)
         assert index2.slice_locs("c", "a") == (2, 6)
+
+
+def test_np_datetime64_objects():
+    # GH#50690
+    ms = np.datetime64(1, "ms")
+    us = np.datetime64(1000, "us")
+
+    left = Index([ms], dtype=object)
+    right = Index([us], dtype=object)
+
+    assert left[0] in right
+    assert right[0] in left
+
+    assert left.get_loc(right[0]) == 0
+    assert right.get_loc(left[0]) == 0
+
+    # non-monotonic cases go through different paths in cython code
+    sec = np.datetime64("9999-01-01", "s")
+    day = np.datetime64("2016-01-01", "D")
+    left2 = Index([ms, sec, day], dtype=object)
+
+    expected = np.array([0], dtype=np.intp)
+    res = left2[:1].get_indexer(right)
+    tm.assert_numpy_array_equal(res, expected)
+    res = left2.get_indexer(right)
+    tm.assert_numpy_array_equal(res, expected)

setup.py

@@ -442,7 +442,8 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
     "_libs.algos": {
         "pyxfile": "_libs/algos",
         "include": klib_include,
-        "depends": _pxi_dep["algos"],
+        "depends": _pxi_dep["algos"] + tseries_depends,
+        "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"],
     },
     "_libs.arrays": {"pyxfile": "_libs/arrays"},
     "_libs.groupby": {"pyxfile": "_libs/groupby"},
@@ -453,21 +454,30 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
         "depends": (
             ["pandas/_libs/src/klib/khash_python.h", "pandas/_libs/src/klib/khash.h"]
             + _pxi_dep["hashtable"]
+            + tseries_depends
         ),
+        "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"],
     },
     "_libs.index": {
         "pyxfile": "_libs/index",
         "include": klib_include,
-        "depends": _pxi_dep["index"],
+        "depends": _pxi_dep["index"] + tseries_depends,
+        "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"],
     },
     "_libs.indexing": {"pyxfile": "_libs/indexing"},
     "_libs.internals": {"pyxfile": "_libs/internals"},
     "_libs.interval": {
         "pyxfile": "_libs/interval",
         "include": klib_include,
-        "depends": _pxi_dep["interval"],
+        "depends": _pxi_dep["interval"] + tseries_depends,
+        "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"],
+    },
+    "_libs.join": {
+        "pyxfile": "_libs/join",
+        "depends": tseries_depends,
+        "include": klib_include,
+        "sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"],
     },
-    "_libs.join": {"pyxfile": "_libs/join", "include": klib_include},
     "_libs.lib": {
         "pyxfile": "_libs/lib",
         "depends": lib_depends + tseries_depends,
@@ -481,10 +491,12 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
         "depends": [
             "pandas/_libs/src/parser/tokenizer.h",
             "pandas/_libs/src/parser/io.h",
-        ],
+        ]
+        + tseries_depends,
         "sources": [
             "pandas/_libs/src/parser/tokenizer.c",
             "pandas/_libs/src/parser/io.c",
+            "pandas/_libs/tslibs/src/datetime/np_datetime.c",
         ],
     },
     "_libs.reduction": {"pyxfile": "_libs/reduction"},