BUG: implement multiple DataFrame.join / merge on non-unique indexes by multiple merges, close #1421

Author: wesm

RELEASE.rst

@@ -182,6 +182,7 @@ pandas 0.8.0
   - Respect column selection for DataFrame in in GroupBy.transform (#1365)
   - Fix MultiIndex partial indexing bug (#1352)
   - Enable assignment of rows in mixed-type DataFrame via .ix (#1432)
+  - Reset index mapping when grouping Series in Cython (#1423)
 
 pandas 0.7.3
 ============

pandas/core/frame.py

@@ -3809,8 +3809,21 @@ def _join_compat(self, other, on=None, how='left', lsuffix='', rsuffix='',
             else:
                 join_axes = None
 
-            return concat([self] + list(other), axis=1, join=how,
-                          join_axes=join_axes, verify_integrity=True)
+            frames = [self] + list(other)
+
+            can_concat = all(df.index.is_unique for df in frames)
+
+            if can_concat:
+                return concat(frames, axis=1, join=how, join_axes=join_axes,
+                              verify_integrity=True)
+
+            joined = frames[0]
+
+            for frame in frames[1:]:
+                joined = merge(joined, frame, how=how,
+                               left_index=True, right_index=True)
+
+            return joined
 
     @Substitution('')
     @Appender(_merge_doc, indents=2)

pandas/tools/merge.py

@@ -347,9 +347,6 @@ def _get_merge_keys(self):
                                                  self.left.index.labels)]
             else:
                 left_keys = [self.left.index.values]
-        # else:
-        #     left_keys.append(self.left.index)
-        #     right_keys.append(self.right.index)
 
         if right_drop:
             self.right = self.right.drop(right_drop, axis=1)
@@ -1039,13 +1036,6 @@ def _get_new_axes(self):
         ndim = self._get_result_dim()
         new_axes = [None] * ndim
 
-        # if self.ignore_index:
-        #     concat_axis = None
-        # else:
-        #     concat_axis = self._get_concat_axis()
-
-        # new_axes[self.axis] = concat_axis
-
         if self.join_axes is None:
             for i in range(ndim):
                 if i == self.axis:

pandas/tools/tests/test_merge.py

@@ -414,6 +414,41 @@ def test_join_float64_float32(self):
         expected = a.join(b.astype('f8'))
         assert_frame_equal(joined, expected)
 
+    def test_join_many_non_unique_index(self):
+        df1 = DataFrame({"a": [1,1], "b": [1,1], "c": [10,20]})
+        df2 = DataFrame({"a": [1,1], "b": [1,2], "d": [100,200]})
+        df3 = DataFrame({"a": [1,1], "b": [1,2], "e": [1000,2000]})
+        idf1 = df1.set_index(["a", "b"])
+        idf2 = df2.set_index(["a", "b"])
+        idf3 = df3.set_index(["a", "b"])
+
+        result = idf1.join([idf2, idf3], how='outer')
+
+        df_partially_merged = merge(df1, df2, on=['a', 'b'], how='outer')
+        expected = merge(df_partially_merged, df3, on=['a', 'b'], how='outer')
+
+        result = result.reset_index()
+
+        result['a'] = result['a'].astype(np.float64)
+        result['b'] = result['b'].astype(np.float64)
+
+        assert_frame_equal(result, expected.ix[:, result.columns])
+
+        df1 = DataFrame({"a": [1, 1, 1], "b": [1,1, 1], "c": [10,20, 30]})
+        df2 = DataFrame({"a": [1, 1, 1], "b": [1,1, 2], "d": [100,200, 300]})
+        df3 = DataFrame({"a": [1, 1, 1], "b": [1,1, 2], "e": [1000,2000, 3000]})
+        idf1 = df1.set_index(["a", "b"])
+        idf2 = df2.set_index(["a", "b"])
+        idf3 = df3.set_index(["a", "b"])
+        result = idf1.join([idf2, idf3], how='inner')
+
+        df_partially_merged = merge(df1, df2, on=['a', 'b'], how='inner')
+        expected = merge(df_partially_merged, df3, on=['a', 'b'], how='inner')
+
+        result = result.reset_index()
+
+        assert_frame_equal(result, expected.ix[:, result.columns])
+
     def test_merge_index_singlekey_right_vs_left(self):
         left = DataFrame({'key': ['a', 'b', 'c', 'd', 'e', 'e', 'a'],
                           'v1': np.random.randn(7)})