From c5a545fd68eb9cbaa859c458755917a6f9b52fb1 Mon Sep 17 00:00:00 2001 From: GYvan <78673871+GYvan@users.noreply.github.com> Date: Mon, 2 Aug 2021 19:00:21 +0200 Subject: [PATCH 1/8] solves issue #40989 by raising NotImplementedError and modifying docs --- pandas/core/generic.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index c63aeb736d16a..58bbfa3b2ef18 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -6155,7 +6155,7 @@ def fillna( NaN values to forward/backward fill. In other words, if there is a gap with more than this number of consecutive NaNs, it will only be partially filled. If method is not specified, this is the - maximum number of entries along the entire axis where NaNs will be + maximum number of entries along the entire index axis where NaNs will be filled. Must be greater than 0 if not None. downcast : dict, default is None A dict of item->dtype of what to downcast if possible, @@ -6314,6 +6314,13 @@ def fillna( else: raise ValueError(f"invalid fill value with a {type(value)}") + if axis == 1 and limit != None: + raise NotImplementedError( + "If limit is specified, " + "values can only be filled " + "along the index axis" + ) + result = self._constructor(new_data) if inplace: return self._update_inplace(result) From bda00fe39b47f243335b74e567acb238746ff506 Mon Sep 17 00:00:00 2001 From: GYvan <78673871+GYvan@users.noreply.github.com> Date: Mon, 2 Aug 2021 19:03:06 +0200 Subject: [PATCH 2/8] Testing the NotImplementedError created to solve issue #40989 --- pandas/tests/generic/test_frame.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pandas/tests/generic/test_frame.py b/pandas/tests/generic/test_frame.py index 49a1dc8bbb21c..c7e8ac7322a39 100644 --- a/pandas/tests/generic/test_frame.py +++ b/pandas/tests/generic/test_frame.py @@ -213,3 +213,22 @@ def test_unexpected_keyword(self): with pytest.raises(TypeError, match=msg): ts.fillna(0, in_place=True) + + def test_fillna_with_columns_and_limit(self): + # 40989 + df = DataFrame( + [ + [np.nan, 2, np.nan, 0], + [3, 4, np.nan, 1], + [np.nan, np.nan, np.nan, 5], + [np.nan, 3, np.nan, 4], + ], + columns=list("ABCD"), + ) + + msg = "If limit is specified, values can only be filled along the index axis" + with pytest.raises(NotImplementedError, match=msg): + df.fillna(axis="columns", value=100, limit=2) + + with pytest.raises(NotImplementedError, match=msg): + df.fillna(axis=1, value=100, limit=1) From 4cba8cafaa773187779e4d2a5b5d80e6378004fa Mon Sep 17 00:00:00 2001 From: GYvan <78673871+GYvan@users.noreply.github.com> Date: Mon, 2 Aug 2021 19:15:25 +0200 Subject: [PATCH 3/8] pre-commit error fixed --- pandas/core/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 58bbfa3b2ef18..569107a53fb6b 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -6314,7 +6314,7 @@ def fillna( else: raise ValueError(f"invalid fill value with a {type(value)}") - if axis == 1 and limit != None: + if axis == 1 and limit is not None: raise NotImplementedError( "If limit is specified, " "values can only be filled " From 44af021abe534f7cf6c5f328f25ec632ec7d4e95 Mon Sep 17 00:00:00 2001 From: GYvan <78673871+GYvan@users.noreply.github.com> Date: Mon, 9 Aug 2021 21:29:36 +0200 Subject: [PATCH 4/8] added modified generic.py --- pandas/core/generic.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 569107a53fb6b..da1478c9a25de 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -6155,7 +6155,7 @@ def fillna( NaN values to forward/backward fill. In other words, if there is a gap with more than this number of consecutive NaNs, it will only be partially filled. If method is not specified, this is the - maximum number of entries along the entire index axis where NaNs will be + maximum number of entries along the entire axis where NaNs will be filled. Must be greater than 0 if not None. downcast : dict, default is None A dict of item->dtype of what to downcast if possible, @@ -6306,21 +6306,25 @@ def fillna( return result if not inplace else None elif not is_list_like(value): - new_data = self._mgr.fillna( - value=value, limit=limit, inplace=inplace, downcast=downcast - ) + if not self._mgr.is_single_block and axis == 1: + + result = self.T.fillna(value=value, limit=limit).T + + # need to downcast here because of all of the transposes + result._mgr = result._mgr.downcast() + + new_data = result + else: + + new_data = self._mgr.fillna( + value=value, limit=limit, inplace=inplace, downcast=downcast + ) elif isinstance(value, ABCDataFrame) and self.ndim == 2: + new_data = self.where(self.notna(), value)._data else: raise ValueError(f"invalid fill value with a {type(value)}") - if axis == 1 and limit is not None: - raise NotImplementedError( - "If limit is specified, " - "values can only be filled " - "along the index axis" - ) - result = self._constructor(new_data) if inplace: return self._update_inplace(result) From 3043fdb858aae5de47e6cc671c0bce8806ca470e Mon Sep 17 00:00:00 2001 From: GYvan <78673871+GYvan@users.noreply.github.com> Date: Mon, 9 Aug 2021 21:30:05 +0200 Subject: [PATCH 5/8] added modified test_fillna.py --- pandas/tests/frame/methods/test_fillna.py | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pandas/tests/frame/methods/test_fillna.py b/pandas/tests/frame/methods/test_fillna.py index 065d074eef6e8..ec993bb993da7 100644 --- a/pandas/tests/frame/methods/test_fillna.py +++ b/pandas/tests/frame/methods/test_fillna.py @@ -438,6 +438,22 @@ def test_fillna_inplace(self): df.fillna(method="ffill", inplace=True) tm.assert_frame_equal(df, expected) + df = DataFrame( + [ + [np.nan, 2, np.nan, 0], + [3, 4, np.nan, 1], + [np.nan, np.nan, np.nan, 5], + [np.nan, 3, np.nan, 4], + ], + columns=list("ABCD"), + ) + + expected = df.fillna(axis=1, value=100, limit=1) + assert expected is not df + + df.fillna(axis=1, value=100, limit=1, inplace=True) + tm.assert_frame_equal(df, expected) + def test_fillna_dict_series(self): df = DataFrame( { @@ -574,6 +590,42 @@ def test_fillna_pos_args_deprecation(self): expected = DataFrame({"a": [1, 2, 3, 0]}, dtype=float) tm.assert_frame_equal(result, expected) + def test_fillna_with_columns_and_limit(self): + # GH40989 + df = DataFrame( + [ + [np.nan, 2, np.nan, 0], + [3, 4, np.nan, 1], + [np.nan, np.nan, np.nan, 5], + [np.nan, 3, np.nan, 4], + ], + columns=list("ABCD"), + ) + result = df.fillna(axis=1, value=100, limit=1) + result2 = df.fillna(axis=1, value=100, limit=2) + + expected = DataFrame( + { + "A": Series([100, 3, 100, 100], dtype="float64"), + "B": [2, 4, np.nan, 3], + "C": [np.nan, 100, np.nan, np.nan], + "D": Series([0, 1, 5, 4], dtype="float64"), + }, + index=[0, 1, 2, 3], + ) + expected2 = DataFrame( + { + "A": Series([100, 3, 100, 100], dtype="float64"), + "B": Series([2, 4, 100, 3], dtype="float64"), + "C": [100, 100, np.nan, 100], + "D": Series([0, 1, 5, 4], dtype="float64"), + }, + index=[0, 1, 2, 3], + ) + + tm.assert_frame_equal(result, expected) + tm.assert_frame_equal(result2, expected2) + def test_fillna_nonconsolidated_frame(): # https://github.com/pandas-dev/pandas/issues/36495 From 91835034e5315bd068beb28052da08409c95b8be Mon Sep 17 00:00:00 2001 From: GYvan <78673871+GYvan@users.noreply.github.com> Date: Mon, 9 Aug 2021 21:40:21 +0200 Subject: [PATCH 6/8] reverted changes for test_frame.py --- pandas/tests/generic/test_frame.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pandas/tests/generic/test_frame.py b/pandas/tests/generic/test_frame.py index c7e8ac7322a39..49a1dc8bbb21c 100644 --- a/pandas/tests/generic/test_frame.py +++ b/pandas/tests/generic/test_frame.py @@ -213,22 +213,3 @@ def test_unexpected_keyword(self): with pytest.raises(TypeError, match=msg): ts.fillna(0, in_place=True) - - def test_fillna_with_columns_and_limit(self): - # 40989 - df = DataFrame( - [ - [np.nan, 2, np.nan, 0], - [3, 4, np.nan, 1], - [np.nan, np.nan, np.nan, 5], - [np.nan, 3, np.nan, 4], - ], - columns=list("ABCD"), - ) - - msg = "If limit is specified, values can only be filled along the index axis" - with pytest.raises(NotImplementedError, match=msg): - df.fillna(axis="columns", value=100, limit=2) - - with pytest.raises(NotImplementedError, match=msg): - df.fillna(axis=1, value=100, limit=1) From ceef973a056b6bab70ce3a6d64fb321a8c76f532 Mon Sep 17 00:00:00 2001 From: GYvan <78673871+GYvan@users.noreply.github.com> Date: Thu, 12 Aug 2021 16:57:36 +0200 Subject: [PATCH 7/8] added another test --- pandas/tests/frame/methods/test_fillna.py | 34 ++++++++++++----------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pandas/tests/frame/methods/test_fillna.py b/pandas/tests/frame/methods/test_fillna.py index ec993bb993da7..1691e8fc7174b 100644 --- a/pandas/tests/frame/methods/test_fillna.py +++ b/pandas/tests/frame/methods/test_fillna.py @@ -438,22 +438,6 @@ def test_fillna_inplace(self): df.fillna(method="ffill", inplace=True) tm.assert_frame_equal(df, expected) - df = DataFrame( - [ - [np.nan, 2, np.nan, 0], - [3, 4, np.nan, 1], - [np.nan, np.nan, np.nan, 5], - [np.nan, 3, np.nan, 4], - ], - columns=list("ABCD"), - ) - - expected = df.fillna(axis=1, value=100, limit=1) - assert expected is not df - - df.fillna(axis=1, value=100, limit=1, inplace=True) - tm.assert_frame_equal(df, expected) - def test_fillna_dict_series(self): df = DataFrame( { @@ -626,6 +610,24 @@ def test_fillna_with_columns_and_limit(self): tm.assert_frame_equal(result, expected) tm.assert_frame_equal(result2, expected2) + def test_fillna_inplace_with_columns_limit_and_value(self): + # GH40989 + df = DataFrame( + [ + [np.nan, 2, np.nan, 0], + [3, 4, np.nan, 1], + [np.nan, np.nan, np.nan, 5], + [np.nan, 3, np.nan, 4], + ], + columns=list("ABCD"), + ) + + expected = df.fillna(axis=1, value=100, limit=1) + assert expected is not df + + df.fillna(axis=1, value=100, limit=1, inplace=True) + tm.assert_frame_equal(df, expected) + def test_fillna_nonconsolidated_frame(): # https://github.com/pandas-dev/pandas/issues/36495 From 49bdfed52749b6b1ec684c9aa4344b47cd96a115 Mon Sep 17 00:00:00 2001 From: GYvan <78673871+GYvan@users.noreply.github.com> Date: Tue, 17 Aug 2021 15:02:23 -0400 Subject: [PATCH 8/8] added into bug fixes in whatsnew --- doc/source/whatsnew/v1.4.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index 8608dffd58090..f9788a3e02019 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -224,7 +224,7 @@ Indexing Missing ^^^^^^^ -- +- Bug in :meth:`DataFrame.fillna` with limit and no method ignores axis='columns' or ``axis = 1`` (:issue:`40989`) - MultiIndex