REGR: revert "CLN: _consolidate_inplace less" / fix regression in fillna() (#34407)

Author: jorisvandenbossche

doc/source/whatsnew/v1.1.5.rst

@@ -21,6 +21,7 @@ Fixed regressions
 - Fixed regression in metadata propagation for ``groupby`` iterator (:issue:`37343`)
 - Fixed regression in indexing on a :class:`Series` with ``CategoricalDtype`` after unpickling (:issue:`37631`)
 - Fixed regression in ``df.groupby(..).rolling(..)`` with the resulting :class:`MultiIndex` when grouping by a label that is in the index (:issue:`37641`)
+- Fixed regression in :meth:`DataFrame.fillna` not filling ``NaN`` after other operations such as :meth:`DataFrame.pivot` (:issue:`36495`).
 
 .. ---------------------------------------------------------------------------
 

pandas/core/generic.py

@@ -3709,6 +3709,8 @@ class   animal   locomotion
         else:
             index = self.index
 
+        self._consolidate_inplace()
+
         if isinstance(index, MultiIndex):
             try:
                 loc, new_index = index._get_loc_level(
@@ -6327,6 +6329,8 @@ def fillna(
         inplace = validate_bool_kwarg(inplace, "inplace")
         value, method = validate_fillna_kwargs(value, method)
 
+        self._consolidate_inplace()
+
         # set the default here, so functions examining the signaure
         # can detect if something was set (e.g. in groupby) (GH9221)
         if axis is None:
@@ -6749,6 +6753,8 @@ def replace(
         if not is_bool(regex) and to_replace is not None:
             raise ValueError("'to_replace' must be 'None' if 'regex' is not a bool")
 
+        self._consolidate_inplace()
+
         if value is None:
             # passing a single value that is scalar like
             # when value is None (GH5319), for compat

pandas/core/internals/managers.py

@@ -442,6 +442,7 @@ def apply(
     def quantile(
         self,
         axis: int = 0,
+        consolidate: bool = True,
         transposed: bool = False,
         interpolation="linear",
         qs=None,
@@ -455,6 +456,8 @@ def quantile(
         Parameters
         ----------
         axis: reduction axis, default 0
+        consolidate: bool, default True. Join together blocks having same
+            dtype
         transposed: bool, default False
             we are holding transposed data
         interpolation : type of interpolation, default 'linear'
@@ -469,6 +472,9 @@ def quantile(
         #  simplify some of the code here and in the blocks
         assert self.ndim >= 2
 
+        if consolidate:
+            self._consolidate_inplace()
+
         def get_axe(block, qs, axes):
             # Because Series dispatches to DataFrame, we will always have
             #  block.ndim == 2

pandas/tests/frame/methods/test_fillna.py

@@ -524,3 +524,18 @@ def test_fill_corner(self, float_frame, float_string_frame):
 
         # TODO(wesm): unused?
         result = empty_float.fillna(value=0)  # noqa
+
+
+def test_fillna_nonconsolidated_frame():
+    # https://github.com/pandas-dev/pandas/issues/36495
+    df = DataFrame(
+        [
+            [1, 1, 1, 1.0],
+            [2, 2, 2, 2.0],
+            [3, 3, 3, 3.0],
+        ],
+        columns=["i1", "i2", "i3", "f1"],
+    )
+    df_nonconsol = df.pivot("i1", "i2")
+    result = df_nonconsol.fillna(0)
+    assert result.isna().sum().sum() == 0