BUG: Hashtable size hint cap (pandas-dev#22805)

Author: chris-b1

doc/source/whatsnew/v0.24.0.rst

@@ -1828,6 +1828,7 @@ Groupby/Resample/Rolling
 - Calling :meth:`pandas.core.groupby.GroupBy.rank` with empty groups and ``pct=True`` was raising a ``ZeroDivisionError`` (:issue:`22519`)
 - Bug in :meth:`DataFrame.resample` when resampling ``NaT`` in ``TimeDeltaIndex`` (:issue:`13223`).
 - Bug in :meth:`DataFrame.groupby` did not respect the ``observed`` argument when selecting a column and instead always used ``observed=False`` (:issue:`23970`)
+- Bug preventing hash table creation with very large number (2^32) of rows (:issue:`22805`)
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/hashtable_class_helper.pxi.in

@@ -262,9 +262,10 @@ dtypes = [('Float64', 'float64', True, 'np.nan'),
 
 cdef class {{name}}HashTable(HashTable):
 
-    def __cinit__(self, size_hint=1):
+    def __cinit__(self, int64_t size_hint=1):
         self.table = kh_init_{{dtype}}()
         if size_hint is not None:
+            size_hint = min(size_hint, _SIZE_HINT_LIMIT)
             kh_resize_{{dtype}}(self.table, size_hint)
 
     def __len__(self):
@@ -573,9 +574,10 @@ cdef class StringHashTable(HashTable):
     # or a sentinel np.nan / None missing value
     na_string_sentinel = '__nan__'
 
-    def __init__(self, int size_hint=1):
+    def __init__(self, int64_t size_hint=1):
         self.table = kh_init_str()
         if size_hint is not None:
+            size_hint = min(size_hint, _SIZE_HINT_LIMIT)
             kh_resize_str(self.table, size_hint)
 
     def __dealloc__(self):
@@ -876,9 +878,11 @@ cdef class StringHashTable(HashTable):
 
 cdef class PyObjectHashTable(HashTable):
 
-    def __init__(self, size_hint=1):
+    def __init__(self, int64_t size_hint=1):
         self.table = kh_init_pymap()
-        kh_resize_pymap(self.table, size_hint)
+        if size_hint is not None:
+            size_hint = min(size_hint, _SIZE_HINT_LIMIT)
+            kh_resize_pymap(self.table, size_hint)
 
     def __dealloc__(self):
         if self.table is not NULL:

pandas/tests/test_algos.py

@@ -1410,6 +1410,14 @@ def test_hashtable_factorize(self, htable, tm_dtype, writable):
         expected_reconstruct = s_duplicated.dropna().values
         tm.assert_numpy_array_equal(result_reconstruct, expected_reconstruct)
 
+    @pytest.mark.parametrize('hashtable', [
+        ht.PyObjectHashTable, ht.StringHashTable,
+        ht.Float64HashTable, ht.Int64HashTable, ht.UInt64HashTable])
+    def test_hashtable_large_sizehint(self, hashtable):
+        # GH 22729
+        size_hint = np.iinfo(np.uint32).max + 1
+        tbl = hashtable(size_hint=size_hint) # noqa
+
 
 def test_quantile():
     s = Series(np.random.randn(100))