taking upper 32bit of PyHash into account as well

Author: realead

asv_bench/benchmarks/hash_functions.py

@@ -25,6 +25,15 @@ def time_isin_outside(self, dtype, exponent):
         self.s.isin(self.values_outside)
 
 
+class UniqueForLargePyObjectInts:
+    def setup(self):
+        lst = [x << 32 for x in range(5000)]
+        self.arr = np.array(lst, dtype=np.object_)
+
+    def time_unique(self):
+        pd.unique(self.arr)
+
+
 class IsinWithRandomFloat:
     params = [
         [np.float64, np.object],

pandas/_libs/src/klib/khash_python.h

@@ -178,11 +178,29 @@ int PANDAS_INLINE pyobject_cmp(PyObject* a, PyObject* b) {
 	return result;
 }
 
-// For PyObject_Hash holds:
-//    hash(0.0) == 0 == hash(-0.0)
-//    hash(X) == 0 if X is a NaN-value
-// so it is OK to use it directly
-#define kh_python_hash_func(key) (PyObject_Hash(key))
+
+khint32_t PANDAS_INLINE kh_python_hash_func(PyObject* key){
+    // For PyObject_Hash holds:
+    //    hash(0.0) == 0 == hash(-0.0)
+    //    hash(X) == 0 if X is a NaN-value
+    // so it is OK to use it directly for doubles
+    Py_hash_t hash = PyObject_Hash(key);
+	if (hash == -1) {
+		PyErr_Clear();
+		return 0;
+	}
+    #if SIZEOF_PY_HASH_T == 4
+        // it is already 32bit value
+        return hash;
+    #else
+        // for 64bit builds,
+        // we need information of the upper 32bits as well
+        // see GH 37615
+        return kh_int64_hash_func(hash);
+    #endif
+}
+
+
 #define kh_python_hash_equal(a, b) (pyobject_cmp(a, b))