BUG: DataFrame.join on keys produce wrong result, does not preserve order

Author: wesm

pandas/src/join.pyx

@@ -51,7 +51,7 @@ def inner_join(ndarray[int32_t] left, ndarray[int32_t] right,
             _get_result_indexer(right_sorter, right_indexer))
 
 def left_outer_join(ndarray[int32_t] left, ndarray[int32_t] right,
-                    Py_ssize_t max_groups):
+                    Py_ssize_t max_groups, sort=True):
     cdef:
         Py_ssize_t i, j, k, count = 0
         ndarray[int32_t] left_count, right_count, left_sorter, right_sorter
@@ -101,8 +101,17 @@ def left_outer_join(ndarray[int32_t] left, ndarray[int32_t] right,
         left_pos += lc
         right_pos += rc
 
-    return (_get_result_indexer(left_sorter, left_indexer),
-            _get_result_indexer(right_sorter, right_indexer))
+    left_indexer = _get_result_indexer(left_sorter, left_indexer)
+    right_indexer = _get_result_indexer(right_sorter, right_indexer)
+
+    if not sort:
+        rev = np.empty(len(left), dtype='i4')
+        rev.put(left_sorter, np.arange(len(left)))
+
+        right_indexer = right_indexer.take(rev)
+        left_indexer = left_indexer.take(rev)
+
+    return left_indexer, right_indexer
 
 
 def full_outer_join(ndarray[int32_t] left, ndarray[int32_t] right,

pandas/tests/test_tseries.py

@@ -80,6 +80,27 @@ def test_left_join_indexer():
     expected = np.array([1, 1, 2, 3, 3], dtype='i4')
     assert(np.array_equal(result, expected))
 
+def test_left_outer_join_bug():
+    left = np.array([0, 1, 0, 1, 1, 2, 3, 1, 0, 2, 1, 2, 0, 1, 1, 2, 3, 2, 3,
+                     2, 1, 1, 3, 0, 3, 2, 3, 0, 0, 2, 3, 2, 0, 3, 1, 3, 0, 1,
+                     3, 0, 0, 1, 0, 3, 1, 0, 1, 0, 1, 1, 0, 2, 2, 2, 2, 2, 0,
+                     3, 1, 2, 0, 0, 3, 1, 3, 2, 2, 0, 1, 3, 0, 2, 3, 2, 3, 3,
+                     2, 3, 3, 1, 3, 2, 0, 0, 3, 1, 1, 1, 0, 2, 3, 3, 1, 2, 0,
+                     3, 1, 2, 0, 2], dtype=np.int32)
+
+    right = np.array([3, 1], dtype=np.int32)
+    max_groups = 4
+
+    lidx, ridx = lib.left_outer_join(left, right, max_groups, sort=False)
+
+    exp_lidx = np.arange(len(left))
+    exp_ridx = -np.ones(len(left))
+    exp_ridx[left == 1] = 1
+    exp_ridx[left == 3] = 0
+
+    assert(np.array_equal(lidx, exp_lidx))
+    assert(np.array_equal(ridx, exp_ridx))
+
 def test_inner_join_indexer():
     a = np.array([1, 2, 3, 4, 5], dtype=np.int64)
     b = np.array([0, 3, 5, 7, 9], dtype=np.int64)

pandas/tools/merge.py

@@ -319,12 +319,12 @@ def _get_multiindex_indexer(join_keys, index, sort=True):
 
     left_group_key, right_group_key, max_groups = \
         _factorize_int64(left_group_key, right_group_key,
-                         sort=sort)
+                         sort=False)
 
     left_indexer, right_indexer = \
         lib.left_outer_join(left_group_key.astype('i4'),
                             right_group_key.astype('i4'),
-                            max_groups)
+                            max_groups, sort=False)
 
     return right_indexer
 

pandas/tools/tests/test_merge.py

@@ -1042,6 +1042,29 @@ def test_concat_single_with_key(self):
         expected = concat([df, df], keys=['foo', 'bar'])
         tm.assert_frame_equal(result, expected[:10])
 
+    def test_left_join_index_preserve_order(self):
+
+        left = DataFrame({'k1' : [0, 1, 2] * 4,
+                          'k2' : ['foo', 'bar'] * 6,
+                          'v' : np.arange(12)})
+
+        index = MultiIndex.from_tuples([(2, 'bar')])
+        right = DataFrame({'v2' : 5}, index=index)
+
+        result = left.join(right, on=['k1', 'k2'])
+
+        expected = left.copy()
+        expected['v2'] = np.nan
+        expected['v2'][(expected.k1 == 2) & (expected.k2 == 'bar')] = 5
+
+        tm.assert_frame_equal(result, expected)
+
+        # do a right join for an extra test
+        joined = merge(right, left, left_index=True,
+                       right_on=['k1', 'k2'], how='right')
+        tm.assert_frame_equal(joined.ix[:, expected.columns], expected)
+
+
 if __name__ == '__main__':
     import nose
     nose.runmodule(argv=[__file__,'-vvs','-x','--pdb', '--pdb-failure'],