From fc4058587fca6dc1840cd8eb687c8acc684f9980 Mon Sep 17 00:00:00 2001 From: phofl Date: Mon, 26 Oct 2020 22:47:21 +0100 Subject: [PATCH 1/6] Fix regression in iloc with boolean list --- doc/source/whatsnew/v1.1.4.rst | 1 + pandas/core/indexing.py | 16 ++++++++-------- pandas/tests/indexing/test_iloc.py | 8 ++++++++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/doc/source/whatsnew/v1.1.4.rst b/doc/source/whatsnew/v1.1.4.rst index eb68ca38ea5b6..7b261d98e7d30 100644 --- a/doc/source/whatsnew/v1.1.4.rst +++ b/doc/source/whatsnew/v1.1.4.rst @@ -24,6 +24,7 @@ Fixed regressions - Fixed regression in :class:`PeriodDtype` comparing both equal and unequal to its string representation (:issue:`37265`) - Fixed regression in certain offsets (:meth:`pd.offsets.Day() ` and below) no longer being hashable (:issue:`37267`) - Fixed regression in :class:`StataReader` which required ``chunksize`` to be manually set when using an iterator to read a dataset (:issue:`37280`) +- Fixed regression in :meth:`DataFrame.iloc.__setitem__` which raised error when trying to set a value after filtering with a boolean list (:issue:`36741`) .. --------------------------------------------------------------------------- diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 27ee91fc5465e..3d491c7127e38 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1670,8 +1670,6 @@ def _setitem_with_indexer(self, indexer, value): "length than the value" ) - pi = plane_indexer[0] if lplane_indexer == 1 else plane_indexer - # we need an iterable, with a ndim of at least 1 # eg. don't pass through np.array(0) if is_list_like_indexer(value) and getattr(value, "ndim", 1) > 0: @@ -1698,7 +1696,7 @@ def _setitem_with_indexer(self, indexer, value): else: v = np.nan - self._setitem_single_column(loc, v, pi) + self._setitem_single_column(loc, v, plane_indexer) elif not unique_cols: raise ValueError( @@ -1716,7 +1714,7 @@ def _setitem_with_indexer(self, indexer, value): else: v = np.nan - self._setitem_single_column(loc, v, pi) + self._setitem_single_column(loc, v, plane_indexer) # we have an equal len ndarray/convertible to our labels # hasattr first, to avoid coercing to ndarray without reason. @@ -1735,7 +1733,9 @@ def _setitem_with_indexer(self, indexer, value): for i, loc in enumerate(ilocs): # setting with a list, re-coerces - self._setitem_single_column(loc, value[:, i].tolist(), pi) + self._setitem_single_column( + loc, value[:, i].tolist(), plane_indexer + ) elif ( len(labels) == 1 @@ -1744,7 +1744,7 @@ def _setitem_with_indexer(self, indexer, value): ): # we have an equal len list/ndarray # We only get here with len(labels) == len(ilocs) == 1 - self._setitem_single_column(ilocs[0], value, pi) + self._setitem_single_column(ilocs[0], value, plane_indexer) elif lplane_indexer == 0 and len(value) == len(self.obj.index): # We get here in one case via .loc with a all-False mask @@ -1759,12 +1759,12 @@ def _setitem_with_indexer(self, indexer, value): ) for loc, v in zip(ilocs, value): - self._setitem_single_column(loc, v, pi) + self._setitem_single_column(loc, v, plane_indexer) else: # scalar value for loc in ilocs: - self._setitem_single_column(loc, value, pi) + self._setitem_single_column(loc, value, plane_indexer) else: self._setitem_single_block(indexer, value) diff --git a/pandas/tests/indexing/test_iloc.py b/pandas/tests/indexing/test_iloc.py index 31abe45215432..9482ed8b3ad8b 100644 --- a/pandas/tests/indexing/test_iloc.py +++ b/pandas/tests/indexing/test_iloc.py @@ -767,3 +767,11 @@ def test_iloc_setitem_series_duplicate_columns(self): ) df.iloc[:, 0] = df.iloc[:, 0].astype(np.float64) assert df.dtypes.iloc[2] == np.int64 + + +def test_iloc_setitem_bool_array(): + # GH: 36741 + df = DataFrame({"flag": ["x", "y", "z"], "value": [1, 3, 4]}) + df.iloc[[True, False, False], 1] = df.iloc[[True, False, False], 1] * 2 + expected = DataFrame({"flag": ["x", "y", "z"], "value": [2, 3, 4]}) + tm.assert_frame_equal(df, expected) From 057130c5f0092a781e0e6718710be7468ab63acb Mon Sep 17 00:00:00 2001 From: phofl Date: Tue, 27 Oct 2020 12:20:16 +0100 Subject: [PATCH 2/6] Move test --- pandas/tests/frame/indexing/test_setitem.py | 7 +++++++ pandas/tests/indexing/test_iloc.py | 8 -------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pandas/tests/frame/indexing/test_setitem.py b/pandas/tests/frame/indexing/test_setitem.py index 87c6ae09aac11..fbb0ce61d3ea7 100644 --- a/pandas/tests/frame/indexing/test_setitem.py +++ b/pandas/tests/frame/indexing/test_setitem.py @@ -213,3 +213,10 @@ def test_setitem_dt64tz(self, timezone_frame): result = df2["B"] tm.assert_series_equal(notna(result), Series([True, False, True], name="B")) tm.assert_series_equal(df2.dtypes, df.dtypes) + + def test_iloc_setitem_bool_array(self): + # GH: 36741 + df = DataFrame({"flag": ["x", "y", "z"], "value": [1, 3, 4]}) + df.iloc[[True, False, False], 1] = df.iloc[[True, False, False], 1] * 2 + expected = DataFrame({"flag": ["x", "y", "z"], "value": [2, 3, 4]}) + tm.assert_frame_equal(df, expected) diff --git a/pandas/tests/indexing/test_iloc.py b/pandas/tests/indexing/test_iloc.py index 9482ed8b3ad8b..31abe45215432 100644 --- a/pandas/tests/indexing/test_iloc.py +++ b/pandas/tests/indexing/test_iloc.py @@ -767,11 +767,3 @@ def test_iloc_setitem_series_duplicate_columns(self): ) df.iloc[:, 0] = df.iloc[:, 0].astype(np.float64) assert df.dtypes.iloc[2] == np.int64 - - -def test_iloc_setitem_bool_array(): - # GH: 36741 - df = DataFrame({"flag": ["x", "y", "z"], "value": [1, 3, 4]}) - df.iloc[[True, False, False], 1] = df.iloc[[True, False, False], 1] * 2 - expected = DataFrame({"flag": ["x", "y", "z"], "value": [2, 3, 4]}) - tm.assert_frame_equal(df, expected) From df7ac24401de004424da06e7170f6644d0154bf2 Mon Sep 17 00:00:00 2001 From: phofl Date: Tue, 27 Oct 2020 12:28:34 +0100 Subject: [PATCH 3/6] Add test again --- pandas/tests/frame/indexing/test_setitem.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/tests/frame/indexing/test_setitem.py b/pandas/tests/frame/indexing/test_setitem.py index c317e90181a8f..8237395008685 100644 --- a/pandas/tests/frame/indexing/test_setitem.py +++ b/pandas/tests/frame/indexing/test_setitem.py @@ -229,3 +229,10 @@ def test_setitem_periodindex(self): rs = df.reset_index().set_index("index") assert isinstance(rs.index, PeriodIndex) tm.assert_index_equal(rs.index, rng) + + def test_iloc_setitem_bool_array(self): + # GH: 36741 + df = DataFrame({"flag": ["x", "y", "z"], "value": [1, 3, 4]}) + df.iloc[[True, False, False], 1] = df.iloc[[True, False, False], 1] * 2 + expected = DataFrame({"flag": ["x", "y", "z"], "value": [2, 3, 4]}) + tm.assert_frame_equal(df, expected) From 52ec52369938bd6f7b1059bbe2ee8c7aaac8d550 Mon Sep 17 00:00:00 2001 From: phofl Date: Tue, 27 Oct 2020 14:09:45 +0100 Subject: [PATCH 4/6] Parametrize test --- pandas/tests/frame/indexing/test_setitem.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/frame/indexing/test_setitem.py b/pandas/tests/frame/indexing/test_setitem.py index 8237395008685..c4d6edc84b8d5 100644 --- a/pandas/tests/frame/indexing/test_setitem.py +++ b/pandas/tests/frame/indexing/test_setitem.py @@ -230,9 +230,11 @@ def test_setitem_periodindex(self): assert isinstance(rs.index, PeriodIndex) tm.assert_index_equal(rs.index, rng) - def test_iloc_setitem_bool_array(self): + @pytest.mark.parametrize("klass", [list, np.array]) + def test_iloc_setitem_bool_array(self, klass): # GH: 36741 df = DataFrame({"flag": ["x", "y", "z"], "value": [1, 3, 4]}) - df.iloc[[True, False, False], 1] = df.iloc[[True, False, False], 1] * 2 + indexer = klass([True, False, False]) + df.iloc[indexer, 1] = df.iloc[indexer, 1] * 2 expected = DataFrame({"flag": ["x", "y", "z"], "value": [2, 3, 4]}) tm.assert_frame_equal(df, expected) From 04d28f8f25ccca3c296bcf700218e300351b8603 Mon Sep 17 00:00:00 2001 From: phofl Date: Tue, 27 Oct 2020 14:10:07 +0100 Subject: [PATCH 5/6] Rename test --- pandas/tests/frame/indexing/test_setitem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/indexing/test_setitem.py b/pandas/tests/frame/indexing/test_setitem.py index c4d6edc84b8d5..55465dffd2027 100644 --- a/pandas/tests/frame/indexing/test_setitem.py +++ b/pandas/tests/frame/indexing/test_setitem.py @@ -231,7 +231,7 @@ def test_setitem_periodindex(self): tm.assert_index_equal(rs.index, rng) @pytest.mark.parametrize("klass", [list, np.array]) - def test_iloc_setitem_bool_array(self, klass): + def test_iloc_setitem_bool_indexer(self, klass): # GH: 36741 df = DataFrame({"flag": ["x", "y", "z"], "value": [1, 3, 4]}) indexer = klass([True, False, False]) From 63d34d40948aec54a68f9300ea8ae445ed9f1c26 Mon Sep 17 00:00:00 2001 From: patrick <61934744+phofl@users.noreply.github.com> Date: Tue, 27 Oct 2020 22:02:26 +0100 Subject: [PATCH 6/6] Update doc/source/whatsnew/v1.1.4.rst Co-authored-by: Joris Van den Bossche --- doc/source/whatsnew/v1.1.4.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.4.rst b/doc/source/whatsnew/v1.1.4.rst index fc2216badb711..332be365e3763 100644 --- a/doc/source/whatsnew/v1.1.4.rst +++ b/doc/source/whatsnew/v1.1.4.rst @@ -26,7 +26,7 @@ Fixed regressions - Fixed regression where slicing :class:`DatetimeIndex` raised :exc:`AssertionError` on irregular time series with ``pd.NaT`` or on unsorted indices (:issue:`36953` and :issue:`35509`) - Fixed regression in certain offsets (:meth:`pd.offsets.Day() ` and below) no longer being hashable (:issue:`37267`) - Fixed regression in :class:`StataReader` which required ``chunksize`` to be manually set when using an iterator to read a dataset (:issue:`37280`) -- Fixed regression in :meth:`DataFrame.iloc.__setitem__` which raised error when trying to set a value after filtering with a boolean list (:issue:`36741`) +- Fixed regression in setitem with :meth:`DataFrame.iloc` which raised error when trying to set a value while filtering with a boolean list (:issue:`36741`) - Fixed regression in :attr:`MultiIndex.is_monotonic_increasing` returning wrong results with ``NaN`` in at least one of the levels (:issue:`37220`) .. ---------------------------------------------------------------------------