From 99d113240552e0ede0cc92e437c3c6143d69c9cb Mon Sep 17 00:00:00 2001 From: Luke Manley Date: Fri, 25 Feb 2022 07:54:45 -0500 Subject: [PATCH 1/5] multiindex slice with negative step --- pandas/core/indexes/multi.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index cc6c92a27e344..286e2b08f7c3a 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -3154,9 +3154,6 @@ def convert_indexer(start, stop, step, indexer=indexer, codes=level_codes): # given the inputs and the codes/indexer, compute an indexer set # if we have a provided indexer, then this need not consider # the entire labels set - if step is not None and step < 0: - # Switch elements for negative step size - start, stop = stop - 1, start - 1 r = np.arange(start, stop, step) if indexer is not None and len(indexer) != len(codes): @@ -3192,14 +3189,20 @@ def convert_indexer(start, stop, step, indexer=indexer, codes=level_codes): try: if key.start is not None: start = level_index.get_loc(key.start) + elif key.step is not None and key.step < 0: + start = len(level_index) - 1 else: start = 0 + if key.stop is not None: stop = level_index.get_loc(key.stop) + elif key.step is not None and key.step < 0: + stop = -1 elif isinstance(start, slice): stop = len(level_index) else: stop = len(level_index) - 1 + step = key.step except KeyError: @@ -3220,7 +3223,9 @@ def convert_indexer(start, stop, step, indexer=indexer, codes=level_codes): elif level > 0 or self._lexsort_depth == 0 or step is not None: # need to have like semantics here to right # searching as when we are using a slice - # so include the stop+1 (so we include stop) + # so adjust the stop by 1 (so we include stop) + if step is not None and step < 0: + return convert_indexer(start, stop - 1, step) return convert_indexer(start, stop + 1, step) else: # sorted, so can return slice object -> view @@ -3512,7 +3517,7 @@ def _reorder_indexer( new_order = key_order_map[self.codes[i][indexer]] elif isinstance(k, slice) and k.step is not None and k.step < 0: - new_order = np.arange(n)[k][indexer] + new_order = np.arange(n)[::-1][indexer] elif isinstance(k, slice) and k.start is None and k.stop is None: # slice(None) should not determine order GH#31330 new_order = np.ones((n,))[indexer] From 86c268ba614ecd32625971b1f38c6807cac2c4af Mon Sep 17 00:00:00 2001 From: Luke Manley Date: Fri, 25 Feb 2022 16:57:53 -0500 Subject: [PATCH 2/5] add tests and cleanup --- pandas/core/indexes/multi.py | 16 +++--- .../tests/indexing/multiindex/test_slice.py | 52 ++++++++++++++++--- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 286e2b08f7c3a..5b89f1ba0dc1d 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -3185,25 +3185,25 @@ def convert_indexer(start, stop, step, indexer=indexer, codes=level_codes): if isinstance(key, slice): # handle a slice, returning a slice if we can # otherwise a boolean indexer + step = key.step + is_negative_step = step is not None and step < 0 try: if key.start is not None: start = level_index.get_loc(key.start) - elif key.step is not None and key.step < 0: + elif is_negative_step: start = len(level_index) - 1 else: start = 0 if key.stop is not None: stop = level_index.get_loc(key.stop) - elif key.step is not None and key.step < 0: - stop = -1 + elif is_negative_step: + stop = 0 elif isinstance(start, slice): stop = len(level_index) else: stop = len(level_index) - 1 - - step = key.step except KeyError: # we have a partial slice (like looking up a partial date @@ -3224,9 +3224,8 @@ def convert_indexer(start, stop, step, indexer=indexer, codes=level_codes): # need to have like semantics here to right # searching as when we are using a slice # so adjust the stop by 1 (so we include stop) - if step is not None and step < 0: - return convert_indexer(start, stop - 1, step) - return convert_indexer(start, stop + 1, step) + stop = (stop - 1) if is_negative_step else (stop + 1) + return convert_indexer(start, stop, step) else: # sorted, so can return slice object -> view i = algos.searchsorted(level_codes, start, side="left") @@ -3517,6 +3516,7 @@ def _reorder_indexer( new_order = key_order_map[self.codes[i][indexer]] elif isinstance(k, slice) and k.step is not None and k.step < 0: + # flip order for negative step new_order = np.arange(n)[::-1][indexer] elif isinstance(k, slice) and k.start is None and k.stop is None: # slice(None) should not determine order GH#31330 diff --git a/pandas/tests/indexing/multiindex/test_slice.py b/pandas/tests/indexing/multiindex/test_slice.py index 55d45a21d643a..91ea1f7cec324 100644 --- a/pandas/tests/indexing/multiindex/test_slice.py +++ b/pandas/tests/indexing/multiindex/test_slice.py @@ -758,12 +758,48 @@ def test_int_series_slicing(self, multiindex_year_month_day_dataframe_random_dat expected = ymd.reindex(s.index[5:]) tm.assert_frame_equal(result, expected) - def test_loc_slice_negative_stepsize(self): + @pytest.mark.parametrize( + "dtype, loc, iloc", + [ + # dtype = int, step = -1 + ("int", slice(None, None, -1), slice(None, None, -1)), + ("int", slice(3, None, -1), slice(3, None, -1)), + ("int", slice(None, 1, -1), slice(None, 0, -1)), + ("int", slice(3, 1, -1), slice(3, 0, -1)), + # dtype = int, step = -2 + ("int", slice(None, None, -2), slice(None, None, -2)), + ("int", slice(3, None, -2), slice(3, None, -2)), + ("int", slice(None, 1, -2), slice(None, 0, -2)), + ("int", slice(3, 1, -2), slice(3, 0, -2)), + # dtype = str, step = -1 + ("str", slice(None, None, -1), slice(None, None, -1)), + ("str", slice("d", None, -1), slice(3, None, -1)), + ("str", slice(None, "b", -1), slice(None, 0, -1)), + ("str", slice("d", "b", -1), slice(3, 0, -1)), + # dtype = str, step = -2 + ("str", slice(None, None, -2), slice(None, None, -2)), + ("str", slice("d", None, -2), slice(3, None, -2)), + ("str", slice(None, "b", -2), slice(None, 0, -2)), + ("str", slice("d", "b", -2), slice(3, 0, -2)), + ], + ) + def test_loc_slice_negative_stepsize(self, dtype, loc, iloc): # GH#38071 - mi = MultiIndex.from_product([["a", "b"], [0, 1]]) - df = DataFrame([[1, 2], [3, 4], [5, 6], [7, 8]], index=mi) - result = df.loc[("a", slice(None, None, -1)), :] - expected = DataFrame( - [[3, 4], [1, 2]], index=MultiIndex.from_tuples([("a", 1), ("a", 0)]) - ) - tm.assert_frame_equal(result, expected) + labels = { + "str": list("abcde"), + "int": range(5), + }[dtype] + + mi = MultiIndex.from_arrays([labels] * 2) + df = DataFrame(1.0, index=mi, columns=["A"]) + + SLC = pd.IndexSlice + + expected = df.iloc[iloc, :] + result_get_loc = df.loc[SLC[loc], :] + result_get_locs_level_0 = df.loc[SLC[loc, :], :] + result_get_locs_level_1 = df.loc[SLC[:, loc], :] + + tm.assert_frame_equal(result_get_loc, expected) + tm.assert_frame_equal(result_get_locs_level_0, expected) + tm.assert_frame_equal(result_get_locs_level_1, expected) From 768c66a849005b1a7d0aefb7e1104992ca6da107 Mon Sep 17 00:00:00 2001 From: Luke Manley Date: Fri, 25 Feb 2022 17:02:43 -0500 Subject: [PATCH 3/5] whatsnew --- doc/source/whatsnew/v1.5.0.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index 199a52f9d770f..b6714e9ede19c 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -349,7 +349,9 @@ Missing MultiIndex ^^^^^^^^^^ -- +- Bug in :meth:`DataFrame.loc` returning empty result when indexer contains a slice with a negative step size and non-null start/stop values (:issue:`...`) +- Bug in :meth:`DataFrame.loc` raising when indexer contains a slice with a negative step size other than -1 (:issue:`...`) +- Bug in :meth:`DataFrame.loc` raising when indexer contains a slice with a negative step size and slicing a non-int labeled index level (:issue:`...`) - I/O From b5f4a99733f6d7112126b7744572c12e8eb128eb Mon Sep 17 00:00:00 2001 From: Luke Manley Date: Fri, 25 Feb 2022 17:08:29 -0500 Subject: [PATCH 4/5] whatsnew --- doc/source/whatsnew/v1.5.0.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index b6714e9ede19c..73a65b51e688f 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -349,9 +349,9 @@ Missing MultiIndex ^^^^^^^^^^ -- Bug in :meth:`DataFrame.loc` returning empty result when indexer contains a slice with a negative step size and non-null start/stop values (:issue:`...`) -- Bug in :meth:`DataFrame.loc` raising when indexer contains a slice with a negative step size other than -1 (:issue:`...`) -- Bug in :meth:`DataFrame.loc` raising when indexer contains a slice with a negative step size and slicing a non-int labeled index level (:issue:`...`) +- Bug in :meth:`DataFrame.loc` returning empty result when indexer contains a slice with a negative step size and non-null start/stop values (:issue:`46156`) +- Bug in :meth:`DataFrame.loc` raising when indexer contains a slice with a negative step size other than -1 (:issue:`46156`) +- Bug in :meth:`DataFrame.loc` raising when indexer contains a slice with a negative step size and slicing a non-int labeled index level (:issue:`46156`) - I/O From e142425af1a4487d17417d0f16ec267a4586c10c Mon Sep 17 00:00:00 2001 From: Luke Manley Date: Thu, 10 Mar 2022 07:11:52 -0500 Subject: [PATCH 5/5] update whatsnew --- doc/source/whatsnew/v1.5.0.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index 11db8c170648d..b01ddaacb2ecf 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -350,9 +350,9 @@ Missing MultiIndex ^^^^^^^^^^ -- Bug in :meth:`DataFrame.loc` returning empty result when indexer contains a slice with a negative step size and non-null start/stop values (:issue:`46156`) -- Bug in :meth:`DataFrame.loc` raising when indexer contains a slice with a negative step size other than -1 (:issue:`46156`) -- Bug in :meth:`DataFrame.loc` raising when indexer contains a slice with a negative step size and slicing a non-int labeled index level (:issue:`46156`) +- Bug in :meth:`DataFrame.loc` returning empty result when slicing a :class:`MultiIndex` with a negative step size and non-null start/stop values (:issue:`46156`) +- Bug in :meth:`DataFrame.loc` raising when slicing a :class:`MultiIndex` with a negative step size other than -1 (:issue:`46156`) +- Bug in :meth:`DataFrame.loc` raising when slicing a :class:`MultiIndex` with a negative step size and slicing a non-int labeled index level (:issue:`46156`) - Bug in :class:`MultiIndex.equals` not commutative when only one side has extension array dtype (:issue:`46026`) -