diff --git a/doc/source/whatsnew/v1.1.5.rst b/doc/source/whatsnew/v1.1.5.rst index a8bbf692a72e5..29b0e99a3a356 100644 --- a/doc/source/whatsnew/v1.1.5.rst +++ b/doc/source/whatsnew/v1.1.5.rst @@ -21,6 +21,7 @@ Fixed regressions - Fixed regression in metadata propagation for ``groupby`` iterator (:issue:`37343`) - Fixed regression in indexing on a :class:`Series` with ``CategoricalDtype`` after unpickling (:issue:`37631`) - Fixed regression in ``df.groupby(..).rolling(..)`` with the resulting :class:`MultiIndex` when grouping by a label that is in the index (:issue:`37641`) +- Fixed regression in :meth:`DataFrame.fillna` not filling ``NaN`` after other operations such as :meth:`DataFrame.pivot` (:issue:`36495`). .. --------------------------------------------------------------------------- diff --git a/pandas/core/generic.py b/pandas/core/generic.py index e866314f00639..d6a2480cf688d 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3709,6 +3709,8 @@ class animal locomotion else: index = self.index + self._consolidate_inplace() + if isinstance(index, MultiIndex): try: loc, new_index = index._get_loc_level( @@ -6327,6 +6329,8 @@ def fillna( inplace = validate_bool_kwarg(inplace, "inplace") value, method = validate_fillna_kwargs(value, method) + self._consolidate_inplace() + # set the default here, so functions examining the signaure # can detect if something was set (e.g. in groupby) (GH9221) if axis is None: @@ -6749,6 +6753,8 @@ def replace( if not is_bool(regex) and to_replace is not None: raise ValueError("'to_replace' must be 'None' if 'regex' is not a bool") + self._consolidate_inplace() + if value is None: # passing a single value that is scalar like # when value is None (GH5319), for compat diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 760765e3a20e6..b13368ca5af18 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -442,6 +442,7 @@ def apply( def quantile( self, axis: int = 0, + consolidate: bool = True, transposed: bool = False, interpolation="linear", qs=None, @@ -455,6 +456,8 @@ def quantile( Parameters ---------- axis: reduction axis, default 0 + consolidate: bool, default True. Join together blocks having same + dtype transposed: bool, default False we are holding transposed data interpolation : type of interpolation, default 'linear' @@ -469,6 +472,9 @@ def quantile( # simplify some of the code here and in the blocks assert self.ndim >= 2 + if consolidate: + self._consolidate_inplace() + def get_axe(block, qs, axes): # Because Series dispatches to DataFrame, we will always have # block.ndim == 2 diff --git a/pandas/tests/frame/methods/test_fillna.py b/pandas/tests/frame/methods/test_fillna.py index bbb57da39705b..d59b70fa91a57 100644 --- a/pandas/tests/frame/methods/test_fillna.py +++ b/pandas/tests/frame/methods/test_fillna.py @@ -524,3 +524,18 @@ def test_fill_corner(self, float_frame, float_string_frame): # TODO(wesm): unused? result = empty_float.fillna(value=0) # noqa + + +def test_fillna_nonconsolidated_frame(): + # https://github.com/pandas-dev/pandas/issues/36495 + df = DataFrame( + [ + [1, 1, 1, 1.0], + [2, 2, 2, 2.0], + [3, 3, 3, 3.0], + ], + columns=["i1", "i2", "i3", "f1"], + ) + df_nonconsol = df.pivot("i1", "i2") + result = df_nonconsol.fillna(0) + assert result.isna().sum().sum() == 0