fixing sizeof for PyObjectHashTable

Author: realead

pandas/_libs/hashtable_class_helper.pxi.in

@@ -996,9 +996,11 @@ cdef class PyObjectHashTable(HashTable):
 
     def sizeof(self, deep=False):
         """ return the size of my table in bytes """
-        return self.table.n_buckets * (sizeof(PyObject *) +  # keys
-                                       sizeof(Py_ssize_t) +  # vals
-                                       sizeof(uint32_t))  # flags
+        overhead = 4 * sizeof(uint32_t) + 3 * sizeof(uint32_t*)
+        for_flags = max(1, self.table.n_buckets >> 5) * sizeof(uint32_t)
+        for_pairs =  self.table.n_buckets * (sizeof(PyObject *) +  # keys
+                                             sizeof(Py_ssize_t))   # vals
+        return overhead + for_flags + for_pairs
 
     cpdef get_item(self, object val):
         cdef:

pandas/tests/libs/test_hashtable.py

@@ -29,6 +29,7 @@ def get_allocated_khash_memory():
 @pytest.mark.parametrize(
     "table_type, dtype",
     [
+        (ht.PyObjectHashTable, np.object),
         (ht.Int64HashTable, np.int64),
         (ht.UInt64HashTable, np.uint64),
         (ht.Float64HashTable, np.float64),
@@ -73,13 +74,15 @@ def test_get_set_contains_len(self, table_type, dtype):
         assert str(index + 2) in str(excinfo.value)
 
     def test_map(self, table_type, dtype):
-        N = 77
-        table = table_type()
-        keys = np.arange(N).astype(dtype)
-        vals = np.arange(N).astype(np.int64) + N
-        table.map(keys, vals)
-        for i in range(N):
-            assert table.get_item(keys[i]) == i + N
+        # PyObjectHashTable has no map-method
+        if table_type != ht.PyObjectHashTable:
+            N = 77
+            table = table_type()
+            keys = np.arange(N).astype(dtype)
+            vals = np.arange(N).astype(np.int64) + N
+            table.map(keys, vals)
+            for i in range(N):
+                assert table.get_item(keys[i]) == i + N
 
     def test_map_locations(self, table_type, dtype):
         N = 8