BUG: don't let np.prod overflow int64

Author: wesm

pandas/core/common.py

@@ -845,7 +845,13 @@ def banner(message):
     bar = '=' * 80
     return '%s\n%s\n%s' % (bar, message, bar)
 
+def _long_prod(vals):
+    result = 1L
+    for x in vals:
+        result *= x
+    return result
 
+    
 class groupby(dict):
     """
     A simple groupby different from the one in itertools.

pandas/core/groupby.py

@@ -2242,7 +2242,7 @@ def _indexer_from_factorized(labels, shape, compress=True):
         max_group = len(obs_ids)
     else:
         comp_ids = group_index
-        max_group = np.prod(shape)
+        max_group = com._long_prod(shape)
 
     if max_group > 1e6:
         # Use mergesort to avoid memory errors in counting sort

pandas/core/reshape.py

@@ -402,7 +402,7 @@ def _unstack_frame(obj, level):
 
 def get_compressed_ids(labels, sizes):
     # no overflow
-    if _long_prod(sizes) < 2 ** 63:
+    if com._long_prod(sizes) < 2 ** 63:
         group_index = get_group_index(labels, sizes)
         comp_index, obs_ids = _compress_group_index(group_index)
     else:
@@ -411,9 +411,9 @@ def get_compressed_ids(labels, sizes):
         for v in labels:
             mask |= v < 0
 
-        while _long_prod(sizes) >= 2 ** 63:
+        while com._long_prod(sizes) >= 2 ** 63:
             i = len(sizes)
-            while _long_prod(sizes[:i]) >= 2 ** 63:
+            while com._long_prod(sizes[:i]) >= 2 ** 63:
                 i -= 1
 
             rem_index, rem_ids = get_compressed_ids(labels[:i],
@@ -426,13 +426,6 @@ def get_compressed_ids(labels, sizes):
     return comp_index, obs_ids
 
 
-def _long_prod(vals):
-    result = 1L
-    for x in vals:
-        result *= x
-    return result
-
-
 def stack(frame, level=-1, dropna=True):
     """
     Convert DataFrame to Series with multi-level Index. Columns become the