ENH calculate size of the resulting pivot table and raise error if it's too big

Author: anhqle

pandas/core/reshape/pivot.py

@@ -31,11 +31,6 @@ def pivot_table(data, values=None, index=None, columns=None, aggfunc='mean',
     index = _convert_by(index)
     columns = _convert_by(columns)
 
-    num_rows = data.reindex(index, axis='columns').shape[0]
-    num_columns = data.reindex(columns, axis='columns').shape[0]
-    if num_rows * num_columns > (2 ** 31 - 1):
-        raise ValueError('Pivot table is too big, causing int32 overflow')
-
     if isinstance(aggfunc, list):
         pieces = []
         keys = []
@@ -86,9 +81,14 @@ def pivot_table(data, values=None, index=None, columns=None, aggfunc='mean',
                 pass
         values = list(values)
 
-    # group by the cartesian product of the grouper
-    # if we have a categorical
-    grouped = data.groupby(keys, observed=False)
+    num_rows = (data.reindex(columns=index).drop_duplicates().shape[0]
+                if index else 1)
+    num_cols = (data.reindex(columns=columns).drop_duplicates().shape[0]
+                if columns else 1)
+    if num_rows * num_cols * len(values) > (2 ** 31 - 1):
+        raise ValueError('Pivot table is too big, causing int32 overflow')
+
+    grouped = data.groupby(keys)
     agged = grouped.agg(aggfunc)
     if dropna and isinstance(agged, ABCDataFrame) and len(agged.columns):
         agged = agged.dropna(how='all')