Skip to content

BUG: merge_asof should be treated as a left join #34484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Apr 12, 2021
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v1.3.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,7 @@ Reshaping
^^^^^^^^^
- Bug in :func:`merge` raising error when performing an inner join with partial index and ``right_index`` when no overlap between indices (:issue:`33814`)
- Bug in :meth:`DataFrame.unstack` with missing levels led to incorrect index names (:issue:`37510`)
- Bug in :func:`merge_asof` propagating the right Index with ``left_index=True`` and ``right_on`` specification instead of left Index (:issue:`33463`)
- Bug in :func:`join` over :class:`MultiIndex` returned wrong result, when one of both indexes had only one level (:issue:`36909`)
- :meth:`merge_asof` raises ``ValueError`` instead of cryptic ``TypeError`` in case of non-numerical merge columns (:issue:`29130`)
- Bug in :meth:`DataFrame.join` not assigning values correctly when having :class:`MultiIndex` where at least one dimension is from dtype ``Categorical`` with non-alphabetically sorted categories (:issue:`38502`)
Expand Down
15 changes: 10 additions & 5 deletions pandas/core/reshape/merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -921,19 +921,25 @@ def _get_join_info(self):
self.left.index,
self.right.index,
left_indexer,
right_indexer,
how="right",
)
else:
join_index = self.right.index.take(right_indexer)
left_indexer = np.array([-1] * len(join_index))
elif self.left_index:
if len(self.right) > 0:
if self.how == "asof":
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you write a comment here explaining why we're doing how="left" here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

join_index = self._create_join_index(
self.left.index,
self.right.index,
left_indexer,
how="left",
)

elif len(self.right) > 0:
join_index = self._create_join_index(
self.right.index,
self.left.index,
right_indexer,
left_indexer,
how="left",
)
else:
Expand All @@ -951,8 +957,7 @@ def _create_join_index(
index: Index,
other_index: Index,
indexer,
other_indexer,
how: str = "left",
how: str,
):
"""
Create a join index by rearranging one index to match another
Expand Down
30 changes: 27 additions & 3 deletions pandas/tests/reshape/merge/test_merge_asof.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pytz

import pandas as pd
from pandas import Timedelta, merge_asof, read_csv, to_datetime
from pandas import Index, Timedelta, merge_asof, read_csv, to_datetime
import pandas._testing as tm
from pandas.core.reshape.merge import MergeError

Expand Down Expand Up @@ -1335,7 +1335,9 @@ def test_merge_index_column_tz(self):
"from_date": index[1:],
"abc": [2.46] * 3 + [2.19],
},
index=pd.Index([1, 2, 3, 4]),
index=pd.date_range(
"2019-10-01 00:30:00", freq="30min", periods=4, tz="UTC"
),
)
tm.assert_frame_equal(result, expected)

Expand All @@ -1348,7 +1350,7 @@ def test_merge_index_column_tz(self):
"abc": [2.46] * 4 + [2.19],
"xyz": [np.nan, 0.9, 0.8, 0.7, 0.6],
},
index=pd.Index([0, 1, 2, 3, 4]),
index=Index([0, 1, 2, 3, 4]),
)
tm.assert_frame_equal(result, expected)

Expand Down Expand Up @@ -1409,3 +1411,25 @@ def test_merge_asof_non_numerical_dtype_object():
left_by="a",
right_by="left_val",
)


@pytest.mark.parametrize(
"kwargs",
[
{"right_index": True, "left_index": True},
{"left_on": "left_time", "right_index": True},
{"left_index": True, "right_on": "right"},
],
)
def test_merge_asof_index_behavior(kwargs):
# GH 33463
index = Index([1, 5, 10], name="test")
left = pd.DataFrame({"left": ["a", "b", "c"], "left_time": [1, 4, 10]}, index=index)
right = pd.DataFrame({"right": [1, 2, 3, 6, 7]}, index=[1, 2, 3, 6, 7])
result = merge_asof(left, right, **kwargs)

expected = pd.DataFrame(
{"left": ["a", "b", "c"], "left_time": [1, 4, 10], "right": [1, 3, 7]},
index=index,
)
tm.assert_frame_equal(result, expected)