BUG: implement 64-bit int overflowing case in merge. close #2690

Author: wesm

RELEASE.rst

@@ -55,6 +55,8 @@ pandas 0.10.1
   - Add ``logx`` option to DataFrame/Series.plot (GH2327_, #2565)
   - Support reading gzipped data from file-like object
   - ``pivot_table`` aggfunc can be anything used in GroupBy.aggregate (GH2643_)
+  - Implement DataFrame merges in case where set cardinalities might overflow
+    64-bit integer (GH2690_)
 
 **Bug fixes**
 
@@ -99,6 +101,7 @@ pandas 0.10.1
 .. _GH2625: https://github.com/pydata/pandas/issues/2625
 .. _GH2643: https://github.com/pydata/pandas/issues/2643
 .. _GH2637: https://github.com/pydata/pandas/issues/2637
+.. _GH2690: https://github.com/pydata/pandas/issues/2690
 .. _GH2692: https://github.com/pydata/pandas/issues/2692
 
 pandas 0.10.0

pandas/tools/merge.py

@@ -425,18 +425,20 @@ def _get_join_indexers(left_keys, right_keys, sort=False, how='inner'):
         right_labels.append(rlab)
         group_sizes.append(count)
 
-    left_group_key = get_group_index(left_labels, group_sizes)
-    right_group_key = get_group_index(right_labels, group_sizes)
-
     max_groups = 1L
     for x in group_sizes:
         max_groups *= long(x)
 
     if max_groups > 2 ** 63:  # pragma: no cover
-        raise MergeError('Combinatorial explosion! (boom)')
+        left_group_key, right_group_key, max_groups = \
+            _factorize_keys(lib.fast_zip(left_labels),
+                            lib.fast_zip(right_labels))
+    else:
+        left_group_key = get_group_index(left_labels, group_sizes)
+        right_group_key = get_group_index(right_labels, group_sizes)
 
-    left_group_key, right_group_key, max_groups = \
-        _factorize_keys(left_group_key, right_group_key, sort=sort)
+        left_group_key, right_group_key, max_groups = \
+            _factorize_keys(left_group_key, right_group_key, sort=sort)
 
     join_func = _join_functions[how]
     return join_func(left_group_key, right_group_key, max_groups)

pandas/tools/tests/test_merge.py

@@ -849,6 +849,16 @@ def test_merge_na_keys(self):
 
         tm.assert_frame_equal(result, expected)
 
+    def test_int64_overflow_issues(self):
+        # #2690, combinatorial explosion
+        df1 = DataFrame(np.random.randn(1000, 7),
+                        columns=list('ABCDEF') + ['G1'])
+        df2 = DataFrame(np.random.randn(1000, 7),
+                        columns=list('ABCDEF') + ['G2'])
+
+        # it works!
+        result = merge(df1, df2, how='outer')
+        self.assertTrue(len(result) == 2000)
 
 def _check_join(left, right, result, join_col, how='left',
                 lsuffix='_x', rsuffix='_y'):