From 04d9f4cb60932374bb8fdd037bab68c42f3c9103 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 26 Sep 2020 15:09:53 +0100 Subject: [PATCH 1/3] REGR: fillna not filling NaNs after pivot without explicitly listing pivot values on 1.1.x --- doc/source/whatsnew/v1.1.3.rst | 1 + pandas/core/internals/managers.py | 2 ++ pandas/tests/frame/test_missing.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/doc/source/whatsnew/v1.1.3.rst b/doc/source/whatsnew/v1.1.3.rst index eded30ca45025..0f908de41a7bb 100644 --- a/doc/source/whatsnew/v1.1.3.rst +++ b/doc/source/whatsnew/v1.1.3.rst @@ -34,6 +34,7 @@ Fixed regressions - Fixed regression when adding a :meth:`timedelta_range` to a :class:``Timestamp`` raised an ``ValueError`` (:issue:`35897`) - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a tuple (:issue:`35534`) - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a frozenset (:issue:`35747`) +- Fixed regression where :meth:`DataFrame.fillna` not filling ``NaN`` after :meth:`DataFrame.pivot` operation (:issue:`36495`) - Fixed regression in :meth:`read_excel` with ``engine="odf"`` caused ``UnboundLocalError`` in some cases where cells had nested child nodes (:issue:`36122`, :issue:`35802`) - Fixed regression in :meth:`DataFrame.replace` inconsistent replace when using a float in the replace method (:issue:`35376`) - Fixed regression in :class:`DataFrame` and :class:`Series` comparisons between numeric arrays and strings (:issue:`35700`, :issue:`36377`) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 3f3f0c68cb1ed..8731e536209d6 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -382,6 +382,8 @@ def apply(self: T, f, align_keys=None, **kwargs) -> T: result_blocks: List[Block] = [] # fillna: Series/DataFrame is responsible for making sure value is aligned + self._consolidate_inplace() + aligned_args = {k: kwargs[k] for k in align_keys} for b in self.blocks: diff --git a/pandas/tests/frame/test_missing.py b/pandas/tests/frame/test_missing.py index b4f91590e09d1..2ae9e12615796 100644 --- a/pandas/tests/frame/test_missing.py +++ b/pandas/tests/frame/test_missing.py @@ -717,3 +717,31 @@ def test_fill_corner(self, float_frame, float_string_frame): # TODO(wesm): unused? result = empty_float.fillna(value=0) # noqa + + def test_fillna_after_pivot(self): + # https://github.com/pandas-dev/pandas/issues/36495 + df = DataFrame( + [ + [1, 1, 1, 1.0], + [2, 2, 2, 2.0], + [3, 3, 3, 3.0], + ], + columns=["i1", "i2", "i3", "f1"], + ) + + result = df.pivot("i1", "i2").fillna(0) + + expected = DataFrame( + [ + [1.0, 0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 2.0, 0.0, 0.0, 2.0, 0.0], + [0.0, 0.0, 3.0, 0.0, 0.0, 3.0], + ], + index=pd.Int64Index([1, 2, 3], dtype="int64", name="i1"), + columns=pd.MultiIndex.from_tuples( + [("i3", 1), ("i3", 2), ("i3", 3), ("f1", 1), ("f1", 2), ("f1", 3)], + names=[None, "i2"], + ), + ) + + tm.assert_frame_equal(result, expected) From edb756c9ce7cc6bfb47f90e2fb912b48f91a46a5 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 26 Sep 2020 16:02:17 +0100 Subject: [PATCH 2/3] reformat with old black --- pandas/tests/frame/test_missing.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/tests/frame/test_missing.py b/pandas/tests/frame/test_missing.py index 2ae9e12615796..e4a5f5760ffbd 100644 --- a/pandas/tests/frame/test_missing.py +++ b/pandas/tests/frame/test_missing.py @@ -721,11 +721,7 @@ def test_fill_corner(self, float_frame, float_string_frame): def test_fillna_after_pivot(self): # https://github.com/pandas-dev/pandas/issues/36495 df = DataFrame( - [ - [1, 1, 1, 1.0], - [2, 2, 2, 2.0], - [3, 3, 3, 3.0], - ], + [[1, 1, 1, 1.0], [2, 2, 2, 2.0], [3, 3, 3, 3.0],], columns=["i1", "i2", "i3", "f1"], ) From cb77328ca9ed7b891d5e9a3390d991640720098d Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 26 Sep 2020 16:05:36 +0100 Subject: [PATCH 3/3] remove comma added by new black not removed with old black --- pandas/tests/frame/test_missing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_missing.py b/pandas/tests/frame/test_missing.py index e4a5f5760ffbd..0e980a6f6fc38 100644 --- a/pandas/tests/frame/test_missing.py +++ b/pandas/tests/frame/test_missing.py @@ -721,7 +721,7 @@ def test_fill_corner(self, float_frame, float_string_frame): def test_fillna_after_pivot(self): # https://github.com/pandas-dev/pandas/issues/36495 df = DataFrame( - [[1, 1, 1, 1.0], [2, 2, 2, 2.0], [3, 3, 3, 3.0],], + [[1, 1, 1, 1.0], [2, 2, 2, 2.0], [3, 3, 3, 3.0]], columns=["i1", "i2", "i3", "f1"], )