Review

Author: harisbal

doc/source/whatsnew/v0.24.0.txt

@@ -169,7 +169,7 @@ This is the same behavior as ``Series.values`` for categorical data. See
 Joining with two multi-indexes
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-As of Pandas 0.24.0 the :func:`Dataframe.join` can be used to join multi-indexed ``Dataframe`` instances on the overlaping index levels (:issue:`20356`)
+As of Pandas 0.24.0 the :func:`Dataframe.join` can be used to join multi-indexed ``Dataframe`` instances on the overlaping index levels (:issue:`6360`)
 
 See the :ref:`Merge, join, and concatenate
 <merging.Join_with_two_multi_indexes>` documentation section.

pandas/core/indexes/base.py

@@ -3912,7 +3912,7 @@ def join(self, other, how='left', level=None, return_indexers=False,
 
     def _join_multi(self, other, how, return_indexers=True):
         from .multi import MultiIndex
-        from pandas.core.reshape.merge import _complete_multilevel_join
+        from pandas.core.reshape.merge import _restore_dropped_levels_multijoin
 
         # figure out join names
         self_names = set(com._not_none(*self.names))
@@ -3928,27 +3928,30 @@ def _join_multi(self, other, how, return_indexers=True):
 
         if self_is_mi and other_is_mi:
 
-            # Drop the non matching levels
-            ldrop_levels = list(set(self_names) - set(overlap))
-            rdrop_levels = list(set(other_names) - set(overlap))
+            # Drop the non-matching levels from left and right respectively
+            ldrop_names = list(set(self_names) - set(overlap))
+            rdrop_names = list(set(other_names) - set(overlap))
 
-            self_jnlevels = self.droplevel(ldrop_levels)
-            other_jnlevels = other.droplevel(rdrop_levels)
-
-            if not (self_jnlevels.is_unique and other_jnlevels.is_unique):
-                raise ValueError("Join on level between two MultiIndex objects"
-                                 "is ambiguous")
-
-            dropped_levels = ldrop_levels + rdrop_levels
+            self_jnlevels = self.droplevel(ldrop_names)
+            other_jnlevels = other.droplevel(rdrop_names)
 
+            # Join left and right
+            # Join on same leveled multi-index frames is supported
             join_idx, lidx, ridx = self_jnlevels.join(other_jnlevels, how,
                                                       return_indexers=True)
 
-            levels, labels, names = _complete_multilevel_join(self, other, how,
-                                                              dropped_levels,
-                                                              join_idx,
-                                                              lidx, ridx)
+            # Restore the dropped levels
+            # Returned index level order is
+            # common levels, ldrop_names, rdrop_names
+            dropped_names = ldrop_names + rdrop_names
+
+            levels, labels, names = (
+                _restore_dropped_levels_multijoin(self, other,
+                                                  dropped_names,
+                                                  join_idx,
+                                                  lidx, ridx))
 
+            # Re-create the multi-index
             multi_join_idx = MultiIndex(levels=levels, labels=labels,
                                         names=names, verify_integrity=False)
 

pandas/core/reshape/merge.py

@@ -1141,12 +1141,12 @@ def _get_join_indexers(left_keys, right_keys, sort=False, how='inner',
     return join_func(lkey, rkey, count, **kwargs)
 
 
-def _complete_multilevel_join(left, right, how, dropped_levels,
-                              join_idx, lidx, ridx):
+def _restore_dropped_levels_multijoin(left, right, dropped_level_names,
+                                      join_idx, lidx, ridx):
     """
     *this is an internal non-public method*
 
-    Returns the levels, labels and names of a multilevel to multilevel join
+    Returns the levels, labels and names of a multil-index to multi-index join.
     Depending on the type of join, this method restores the appropriate
     dropped levels of the joined multi-index. The method relies on lidx, ridx
     which hold the index positions of left and right, where a join was feasible
@@ -1157,19 +1157,18 @@ def _complete_multilevel_join(left, right, how, dropped_levels,
         left index
     right : Index
         right index
+    dropped_level_names : str array
+        list of non-common levels
     join_idx : Index
         the index of the join between the common levels of left and right
-    how : {'left', 'right', 'outer', 'inner'}
     lidx : intp array
         left indexer
     right : intp array
         right indexer
-    dropped_levels : str array
-        list of non-common levels
 
     Returns
     -------
-    levels : intp array
+    levels : intp ndarray
         levels of combined multiindexes
     labels : str array
         labels of combined multiindexes
@@ -1178,12 +1177,20 @@ def _complete_multilevel_join(left, right, how, dropped_levels,
 
     """
 
+    # Convert to 1 level multi-index if not
+    if not isinstance(join_idx, MultiIndex):
+        levels = [join_idx.values]
+        labels = [list(range(0, len(join_idx)))]
+        names = [join_idx.name]
+        join_idx = MultiIndex(levels=levels, labels=labels,
+                              names=names, verify_integrity=False)
+
     join_levels = join_idx.levels
     join_labels = join_idx.labels
     join_names = join_idx.names
 
     # lidx and ridx hold the indexes where the join occured
-    # for left and right respectively. If left (right) is None it means that
+    # for left and right respectively. If left (right) is None then
     # the join occured on all indices of left (right)
     if lidx is None:
         lidx = range(0, len(left))
@@ -1192,27 +1199,26 @@ def _complete_multilevel_join(left, right, how, dropped_levels,
         ridx = range(0, len(right))
 
     # Iterate through the levels that must be restored
-    for dl in dropped_levels:
-        if dl in left.names:
+    for dropped_level_name in dropped_level_names:
+        if dropped_level_name in left.names:
             idx = left
             indexer = lidx
         else:
             idx = right
             indexer = ridx
 
         # The index of the level name to be restored
-        name_idx = idx.names.index(dl)
+        name_idx = idx.names.index(dropped_level_name)
 
         restore_levels = idx.levels[name_idx].values
-        restore_labels = idx.labels[name_idx]
-
-        join_levels = join_levels.__add__([restore_levels])
-        join_names = join_names.__add__([dl])
-
         # Inject -1 in the labels list where a join was not possible
         # IOW indexer[i]=-1
-        labels = [restore_labels[i] if i != -1 else -1 for i in indexer]
-        join_labels = join_labels.__add__([labels])
+        labels = idx.labels[name_idx]
+        restore_labels = [labels[i] if i != -1 else -1 for i in indexer]
+
+        join_levels = join_levels.__add__([restore_levels])
+        join_labels = join_labels.__add__([restore_labels])
+        join_names = join_names.__add__([dropped_level_name])
 
     return join_levels, join_labels, join_names
 

pandas/tests/reshape/merge/test_multi.py

@@ -13,47 +13,50 @@
 from pandas.compat import lzip
 from pandas.core.reshape.concat import concat
 from pandas.core.reshape.merge import merge
-from pandas.util.testing import assert_frame_equal
+
+
+@pytest.fixture
+def left():
+    # a little relevant example with NAs
+    key1 = ['bar', 'bar', 'bar', 'foo', 'foo', 'baz', 'baz', 'qux',
+            'qux', 'snap']
+    key2 = ['two', 'one', 'three', 'one', 'two', 'one', 'two', 'two',
+            'three', 'one']
+
+    data = np.random.randn(len(key1))
+    return DataFrame({'key1': key1, 'key2': key2, 'data': data})
+
+
+@pytest.fixture
+def right():
+    index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
+                               ['one', 'two', 'three']],
+                       labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
+                               [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
+                       names=['first', 'second'])
+
+    return DataFrame(np.random.randn(10, 3), index=index,
+                     columns=['j_one', 'j_two', 'j_three'])
 
 
 class TestMergeMulti(object):
 
-    def setup_method(self):
-        self.index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
-                                        ['one', 'two', 'three']],
-                                labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
-                                        [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
-                                names=['first', 'second'])
-        self.to_join = DataFrame(np.random.randn(10, 3), index=self.index,
-                                 columns=['j_one', 'j_two', 'j_three'])
-
-        # a little relevant example with NAs
-        key1 = ['bar', 'bar', 'bar', 'foo', 'foo', 'baz', 'baz', 'qux',
-                'qux', 'snap']
-        key2 = ['two', 'one', 'three', 'one', 'two', 'one', 'two', 'two',
-                'three', 'one']
-
-        data = np.random.randn(len(key1))
-        self.data = DataFrame({'key1': key1, 'key2': key2,
-                               'data': data})
-
-    def test_merge_on_multikey(self):
-        joined = self.data.join(self.to_join, on=['key1', 'key2'])
-
-        join_key = Index(lzip(self.data['key1'], self.data['key2']))
-        indexer = self.to_join.index.get_indexer(join_key)
-        ex_values = self.to_join.values.take(indexer, axis=0)
+    def test_merge_on_multikey(self, left, right):
+        joined = left.join(right, on=['key1', 'key2'])
+
+        join_key = Index(lzip(left['key1'], left['key2']))
+        indexer = right.index.get_indexer(join_key)
+        ex_values = right.values.take(indexer, axis=0)
         ex_values[indexer == -1] = np.nan
-        expected = self.data.join(DataFrame(ex_values,
-                                            columns=self.to_join.columns))
+        expected = left.join(DataFrame(ex_values, columns=right.columns))
 
         # TODO: columns aren't in the same order yet
-        assert_frame_equal(joined, expected.loc[:, joined.columns])
+        tm.assert_frame_equal(joined, expected.loc[:, joined.columns])
 
-        left = self.data.join(self.to_join, on=['key1', 'key2'], sort=True)
+        left = left.join(right, on=['key1', 'key2'], sort=True)
         right = expected.loc[:, joined.columns].sort_values(['key1', 'key2'],
                                                             kind='mergesort')
-        assert_frame_equal(left, right)
+        tm.assert_frame_equal(left, right)
 
     def test_left_join_multi_index(self):
         icols = ['1st', '2nd', '3rd']
@@ -119,18 +122,18 @@ def run_asserts(left, right):
 
         run_asserts(left, right)
 
-    def test_merge_right_vs_left(self):
+    def test_merge_right_vs_left(self, left, right):
         # compare left vs right merge with multikey
         for sort in [False, True]:
-            merged1 = self.data.merge(self.to_join, left_on=['key1', 'key2'],
-                                      right_index=True, how='left', sort=sort)
+            merged1 = left.merge(right, left_on=['key1', 'key2'],
+                                 right_index=True, how='left', sort=sort)
 
-            merged2 = self.to_join.merge(self.data, right_on=['key1', 'key2'],
-                                         left_index=True, how='right',
-                                         sort=sort)
+            merged2 = right.merge(left, right_on=['key1', 'key2'],
+                                  left_index=True, how='right',
+                                  sort=sort)
 
             merged2 = merged2.loc[:, merged1.columns]
-            assert_frame_equal(merged1, merged2)
+            tm.assert_frame_equal(merged1, merged2)
 
     def test_compress_group_combinations(self):
 
@@ -393,15 +396,13 @@ def test_join_multi_levels(self):
                                'nl0000289965']))
             .set_index(['household_id', 'asset_id'])
             .reindex(columns=['male', 'wealth', 'name', 'share']))
-        assert_frame_equal(result, expected)
-
-        assert_frame_equal(result, expected)
+        tm.assert_frame_equal(result, expected)
 
         # equivalency
-        result2 = (merge(household.reset_index(), portfolio.reset_index(),
+        result = (merge(household.reset_index(), portfolio.reset_index(),
                          on=['household_id'], how='inner')
                    .set_index(['household_id', 'asset_id']))
-        assert_frame_equal(result2, expected)
+        tm.assert_frame_equal(result, expected)
 
         result = household.join(portfolio, how='outer')
         expected = (concat([
@@ -412,7 +413,7 @@ def test_join_multi_levels(self):
                     [(4, np.nan)],
                     names=['household_id', 'asset_id'])))
         ], axis=0, sort=True).reindex(columns=expected.columns))
-        assert_frame_equal(result, expected)
+        tm.assert_frame_equal(result, expected)
 
         # invalid cases
         household.index.name = 'foo'
@@ -471,7 +472,7 @@ def test_join_multi_levels2(self):
         result = (merge(household.reset_index(), log_return.reset_index(),
                         on=['asset_id'], how='inner')
                   .set_index(['household_id', 'asset_id', 't']))
-        assert_frame_equal(result, expected)
+        tm.assert_frame_equal(result, expected)
 
         expected = (
             DataFrame(dict(
@@ -496,7 +497,7 @@ def test_join_multi_levels2(self):
                   on=['asset_id'], how='outer')
                   .set_index(['household_id', 'asset_id', 't']))
 
-        assert_frame_equal(result, expected)
+        tm.assert_frame_equal(result, expected)
 
 
 @pytest.fixture
@@ -564,17 +565,17 @@ def test_join_multi_empty_frames(self, left_multi, right_multi, join_type,
         result = left_multi.join(right_multi, how=join_type).sort_index()
         tm.assert_frame_equal(result, expected)
 
-    @pytest.mark.parametrize("klass", [None, np.asarray, Series, Index])
-    def test_merge_datetime_index(self, klass):
+    @pytest.mark.parametrize("box", [None, np.asarray, Series, Index])
+    def test_merge_datetime_index(self, box):
         # see gh-19038
         df = DataFrame([1, 2, 3],
                        ["2016-01-01", "2017-01-01", "2018-01-01"],
                        columns=["a"])
         df.index = pd.to_datetime(df.index)
         on_vector = df.index.year
 
-        if klass is not None:
-            on_vector = klass(on_vector)
+        if box is not None:
+            on_vector = box(on_vector)
 
         expected = DataFrame(
             OrderedDict([
@@ -596,3 +597,26 @@ def test_merge_datetime_index(self, klass):
 
         result = df.merge(df, on=[df.index.year], how="inner")
         tm.assert_frame_equal(result, expected)
+
+    def test_single_common_level(self):
+        index_left = pd.MultiIndex.from_tuples([('K0', 'X0'), ('K0', 'X1'),
+                                                ('K1', 'X2')],
+                                               names=['key', 'X'])
+
+        left = pd.DataFrame({'A': ['A0', 'A1', 'A2'],
+                             'B': ['B0', 'B1', 'B2']},
+                            index=index_left)
+
+        index_right = pd.MultiIndex.from_tuples([('K0', 'Y0'), ('K1', 'Y1'),
+                                                 ('K2', 'Y2'), ('K2', 'Y3')],
+                                                names=['key', 'Y'])
+
+        right = pd.DataFrame({'C': ['C0', 'C1', 'C2', 'C3'],
+                              'D': ['D0', 'D1', 'D2', 'D3']},
+                             index=index_right)
+
+        result = left.join(right)
+        expected = pd.merge(left.reset_index(), right.reset_index(),
+                            on=['key'], how='inner').set_index(['key', 'X', 'Y'])
+
+        tm.assert_frame_equal(result, expected)