From b5b466ee2ccafcb16c39cc93a1f739477704b1ea Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 27 May 2020 08:59:36 +0200 Subject: [PATCH 1/4] Revert "CLN: _consolidate_inplace less (#34389)" This reverts commit 760ba37a776947694ec816eaf29ce8937e08f544. --- pandas/core/generic.py | 9 +++++++++ pandas/core/internals/managers.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 79805bec85af0..8aa8f8bb60654 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3369,6 +3369,8 @@ class max_speed nv.validate_take(tuple(), kwargs) + self._consolidate_inplace() + new_data = self._mgr.take( indices, axis=self._get_block_manager_axis(axis), verify=True ) @@ -3506,6 +3508,8 @@ class animal locomotion if axis == 1: return self[key] + self._consolidate_inplace() + index = self.index if isinstance(index, MultiIndex): loc, new_index = self.index.get_loc_level(key, drop_level=drop_level) @@ -5428,6 +5432,7 @@ def values(self) -> np.ndarray: ['lion', 80.5, 1], ['monkey', nan, None]], dtype=object) """ + self._consolidate_inplace() return self._mgr.as_array(transpose=self._AXIS_REVERSED) @property @@ -6096,6 +6101,8 @@ def fillna( inplace = validate_bool_kwarg(inplace, "inplace") value, method = validate_fillna_kwargs(value, method) + self._consolidate_inplace() + # set the default here, so functions examining the signaure # can detect if something was set (e.g. in groupby) (GH9221) if axis is None: @@ -6530,6 +6537,8 @@ def replace( if not is_bool(regex) and to_replace is not None: raise AssertionError("'to_replace' must be 'None' if 'regex' is not a bool") + self._consolidate_inplace() + if value is None: # passing a single value that is scalar like # when value is None (GH5319), for compat diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index e43c7bf887bcc..6a7df1c8d88a1 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -378,6 +378,8 @@ def apply(self: T, f, align_keys=None, **kwargs) -> T: result_blocks: List[Block] = [] # fillna: Series/DataFrame is responsible for making sure value is aligned + self._consolidate_inplace() + aligned_args = {k: kwargs[k] for k in align_keys} for b in self.blocks: @@ -410,6 +412,7 @@ def apply(self: T, f, align_keys=None, **kwargs) -> T: def quantile( self, axis: int = 0, + consolidate: bool = True, transposed: bool = False, interpolation="linear", qs=None, @@ -423,6 +426,8 @@ def quantile( Parameters ---------- axis: reduction axis, default 0 + consolidate: bool, default True. Join together blocks having same + dtype transposed: bool, default False we are holding transposed data interpolation : type of interpolation, default 'linear' @@ -437,6 +442,9 @@ def quantile( # simplify some of the code here and in the blocks assert self.ndim >= 2 + if consolidate: + self._consolidate_inplace() + def get_axe(block, qs, axes): # Because Series dispatches to DataFrame, we will always have # block.ndim == 2 @@ -667,6 +675,8 @@ def is_mixed_type(self) -> bool: @property def is_numeric_mixed_type(self) -> bool: + # Warning, consolidation needs to get checked upstairs + self._consolidate_inplace() return all(block.is_numeric for block in self.blocks) @property From 1228510e120e98320fc0a6a566d4051df9347631 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 10 Jun 2020 10:01:12 +0200 Subject: [PATCH 2/4] re-revert take and numeric_mixed_type --- pandas/core/generic.py | 2 -- pandas/core/internals/managers.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 8aa8f8bb60654..54bf76990ff03 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3369,8 +3369,6 @@ class max_speed nv.validate_take(tuple(), kwargs) - self._consolidate_inplace() - new_data = self._mgr.take( indices, axis=self._get_block_manager_axis(axis), verify=True ) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 6a7df1c8d88a1..199148e02ee23 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -675,8 +675,6 @@ def is_mixed_type(self) -> bool: @property def is_numeric_mixed_type(self) -> bool: - # Warning, consolidation needs to get checked upstairs - self._consolidate_inplace() return all(block.is_numeric for block in self.blocks) @property From f60b95d7668a32271a949b8287837d1cd1cd6612 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 26 Sep 2020 18:05:04 +0100 Subject: [PATCH 3/4] remove _consolidate_inplace() causing failures --- pandas/core/internals/managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 854c4b2651e01..3a8fde96d39b7 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -372,8 +372,6 @@ def apply( result_blocks: List[Block] = [] # fillna: Series/DataFrame is responsible for making sure value is aligned - self._consolidate_inplace() - aligned_args = {k: kwargs[k] for k in align_keys} for b in self.blocks: From a5df709c4efb49b5e1fe8aa86756e55784390fe9 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 26 Nov 2020 15:44:34 +0100 Subject: [PATCH 4/4] add whatsnew --- doc/source/whatsnew/v1.1.5.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.1.5.rst b/doc/source/whatsnew/v1.1.5.rst index a8bbf692a72e5..29b0e99a3a356 100644 --- a/doc/source/whatsnew/v1.1.5.rst +++ b/doc/source/whatsnew/v1.1.5.rst @@ -21,6 +21,7 @@ Fixed regressions - Fixed regression in metadata propagation for ``groupby`` iterator (:issue:`37343`) - Fixed regression in indexing on a :class:`Series` with ``CategoricalDtype`` after unpickling (:issue:`37631`) - Fixed regression in ``df.groupby(..).rolling(..)`` with the resulting :class:`MultiIndex` when grouping by a label that is in the index (:issue:`37641`) +- Fixed regression in :meth:`DataFrame.fillna` not filling ``NaN`` after other operations such as :meth:`DataFrame.pivot` (:issue:`36495`). .. ---------------------------------------------------------------------------