From a444a8a741e15e725b52cf896bfc229c106ca2c5 Mon Sep 17 00:00:00 2001 From: phofl Date: Sat, 21 Nov 2020 02:23:08 +0100 Subject: [PATCH 1/5] Fix bug with ordering --- pandas/core/indexes/multi.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 9eb34d920a328..7edc16495897e 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -3156,7 +3156,7 @@ def _update_indexer(idxr: Optional[Index], indexer: Optional[Index]) -> Index: self._get_level_indexer(x, level=i, indexer=indexer) ) indexers = (idxrs if indexers is None else indexers).union( - idxrs + idxrs, sort=False ) except KeyError: @@ -3241,6 +3241,9 @@ def _reorder_indexer( # order they appears in a list-like sequence # This mapping is then use to reorder the indexer for i, k in enumerate(seq): + if is_scalar(k): + # GH#34603 we want to treat a scalar the same as an all equal list + k = [k] if com.is_bool_indexer(k): new_order = np.arange(n)[indexer] elif is_list_like(k): From 5a2e975296e8677b05e7b36fcbc3ee07a8b8704b Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 22 Nov 2020 01:58:29 +0100 Subject: [PATCH 2/5] Fix second case and add test --- pandas/core/indexes/multi.py | 3 +++ pandas/tests/indexing/multiindex/test_loc.py | 27 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 7edc16495897e..9dd5995dcf499 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -3257,6 +3257,9 @@ def _reorder_indexer( key_order_map[level_indexer] = np.arange(len(level_indexer)) new_order = key_order_map[self.codes[i][indexer]] + elif isinstance(k, slice) and k.start is None and k.stop is None: + # slice(None) should not determine order GH#31330 + new_order = np.ones((n,)) else: # For all other case, use the same order as the level new_order = np.arange(n)[indexer] diff --git a/pandas/tests/indexing/multiindex/test_loc.py b/pandas/tests/indexing/multiindex/test_loc.py index 165d34180dfab..a6e270c6414cf 100644 --- a/pandas/tests/indexing/multiindex/test_loc.py +++ b/pandas/tests/indexing/multiindex/test_loc.py @@ -640,3 +640,30 @@ def test_getitem_non_found_tuple(): ) with pytest.raises(KeyError, match=r"\(2\.0, 2\.0, 3\.0\)"): df.loc[(2.0, 2.0, 3.0)] + + +def test_loc_setitem_indexer_differently_ordered(): + # GH#34603 + mi = MultiIndex.from_product([["a", "b"], [0, 1]]) + df = DataFrame([[1, 2], [3, 4], [5, 6], [7, 8]], index=mi) + + indexer = ("a", [1, 0]) + df.loc[indexer, :] = np.array([[9, 10], [11, 12]]) + expected = DataFrame([[11, 12], [9, 10], [5, 6], [7, 8]], index=mi) + tm.assert_frame_equal(df, expected) + + +def test_loc_getitem_index_differently_ordered_slice_none(): + # GH#31330 + df = DataFrame( + [[1, 2], [3, 4], [5, 6], [7, 8]], + index=[["a", "a", "b", "b"], [1, 2, 1, 2]], + columns=["a", "b"], + ) + result = df.loc[(slice(None), [2, 1]), :] + expected = DataFrame( + [[3, 4], [7, 8], [1, 2], [5, 6]], + index=[["a", "b", "a", "b"], [2, 2, 1, 1]], + columns=["a", "b"], + ) + tm.assert_frame_equal(result, expected) From 39c0bf7ed9b0670daf8ea27c4cbc50ecb5f97066 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 22 Nov 2020 02:00:02 +0100 Subject: [PATCH 3/5] Add whatsnew --- doc/source/whatsnew/v1.2.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index b93f98ab0274f..a6721d3ae85e5 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -612,6 +612,7 @@ Indexing - Bug in :meth:`DataFrame.reindex` raising ``IndexingError`` wrongly for empty :class:`DataFrame` with ``tolerance`` not None or ``method="nearest"`` (:issue:`27315`) - Bug in indexing on a :class:`Series` or :class:`DataFrame` with a :class:`CategoricalIndex` using listlike indexer that contains elements that are in the index's ``categories`` but not in the index itself failing to raise ``KeyError`` (:issue:`37901`) - Bug in :meth:`DataFrame.iloc` and :meth:`Series.iloc` aligning objects in ``__setitem__`` (:issue:`22046`) +- Bug in :meth:`DataFrame.loc` returning and assigning elements in wrong order when indexer is differently ordered than the object to filter (:issue:`31330`, :issue:`34603`) Missing ^^^^^^^ From cf6f99933524be39bbcaf55e7ca402e0d18bda02 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 22 Nov 2020 02:02:31 +0100 Subject: [PATCH 4/5] Change whatsnew --- doc/source/whatsnew/v1.2.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index c3a9fc863a6c6..6aa6bc343a94e 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -613,7 +613,7 @@ Indexing - Bug in indexing on a :class:`Series` or :class:`DataFrame` with a :class:`CategoricalIndex` using listlike indexer that contains elements that are in the index's ``categories`` but not in the index itself failing to raise ``KeyError`` (:issue:`37901`) - Bug in :meth:`DataFrame.iloc` and :meth:`Series.iloc` aligning objects in ``__setitem__`` (:issue:`22046`) - Bug in :meth:`DataFrame.loc` did not raise ``KeyError`` when missing combination was given with ``slice(None)`` for remaining levels (:issue:`19556`) -- Bug in :meth:`DataFrame.loc` returning and assigning elements in wrong order when indexer is differently ordered than the object to filter (:issue:`31330`, :issue:`34603`) +- Bug in :meth:`DataFrame.loc` returning and assigning elements in wrong order when indexer is differently ordered than the :class:`MultiIndex` to filter (:issue:`31330`, :issue:`34603`) Missing ^^^^^^^ From 7d0f2f0eb28e56ad1af18fcdce1ec77b7bc966e8 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 22 Nov 2020 21:31:52 +0100 Subject: [PATCH 5/5] Fix bug --- pandas/core/indexes/multi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 4b8622645ee4f..f5917b8a97ffa 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -3247,7 +3247,7 @@ def _reorder_indexer( new_order = key_order_map[self.codes[i][indexer]] elif isinstance(k, slice) and k.start is None and k.stop is None: # slice(None) should not determine order GH#31330 - new_order = np.ones((n,)) + new_order = np.ones((n,))[indexer] else: # For all other case, use the same order as the level new_order = np.arange(n)[indexer]