ENH GH20601 raise error when pivot table's number of levels > int32

doc/source/whatsnew/v0.23.0.txt

@@ -1177,6 +1177,7 @@ Reshaping
 - Bug in :meth:`DataFrame.astype` where column metadata is lost when converting to categorical or a dictionary of dtypes (:issue:`19920`)
 - Bug in :func:`cut` and :func:`qcut` where timezone information was dropped (:issue:`19872`)
 - Bug in :class:`Series` constructor with a ``dtype=str``, previously raised in some cases (:issue:`19853`)
+- Improved error message when the number of levels in a pivot table is too large causing int32 overflow (:issue:`20601`)
 
 Other
 ^^^^^

pandas/core/reshape/reshape.py

@@ -162,6 +162,8 @@ def _make_selectors(self):
         self.full_shape = ngroups, stride
 
         selector = self.sorted_labels[-1] + stride * comp_index + self.lift
+        if np.prod(self.full_shape) > (2 ** 31 - 1):
+            raise ValueError('Pivot table is too big, causing int32 overflow')
         mask = np.zeros(np.prod(self.full_shape), dtype=bool)
         mask.put(selector, True)
 

pandas/tests/reshape/test_pivot.py

@@ -1237,6 +1237,16 @@ def test_pivot_string_func_vs_func(self, f, f_numpy):
                                aggfunc=f_numpy)
         tm.assert_frame_equal(result, expected)
 
+    @pytest.mark.slow
+    def test_pivot_number_of_levels_larger_than_int32(self):
+        # GH 20601
+        data = DataFrame({'ind1': list(range(1337600)) * 2,
+                          'ind2': list(range(3040)) * 2 * 440,
+                          'count': [1] * 2 * 1337600})
+        with tm.assert_raises_regex(ValueError, 'int32 overflow'):
+            data.pivot_table(index='ind1', columns='ind2',
+                             values='count', aggfunc='count')
+
 
 class TestCrosstab(object):