From f78621a104b7e02f3162888abf5bcb309942f334 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 19 Mar 2020 14:47:41 +0100 Subject: [PATCH 1/2] PERF: skip non-consolidatable blocks when checking consolidation --- pandas/core/internals/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index da334561385d6..ada065bc304ff 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -676,7 +676,7 @@ def is_consolidated(self) -> bool: return self._is_consolidated def _consolidate_check(self) -> None: - ftypes = [blk.ftype for blk in self.blocks] + ftypes = [blk.ftype for blk in self.blocks if blk._can_consolidate] self._is_consolidated = len(ftypes) == len(set(ftypes)) self._known_consolidated = True From 72c01e058cf0f27d52db5843a4ab568e59892243 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 19 Mar 2020 17:19:18 +0100 Subject: [PATCH 2/2] remove ftypes --- pandas/core/arrays/sparse/array.py | 1 - pandas/core/internals/blocks.py | 13 ------------- pandas/core/internals/managers.py | 4 ++-- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py index 963c2f3d53138..8021e0babe4e0 100644 --- a/pandas/core/arrays/sparse/array.py +++ b/pandas/core/arrays/sparse/array.py @@ -268,7 +268,6 @@ class SparseArray(PandasObject, ExtensionArray, ExtensionOpsMixin): Indices: array([2, 3], dtype=int32) """ - _pandas_ftype = "sparse" _subtyp = "sparse_array" # register ABCSparseArray _deprecations = PandasObject._deprecations | frozenset(["get_values"]) _sparse_index: SparseIndex diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index adeb1ae04a58d..fec8639f5a44d 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -110,7 +110,6 @@ class Block(PandasObject): _can_consolidate = True _verify_integrity = True _validate_ndim = True - _ftype = "dense" _concatenator = staticmethod(np.concatenate) def __init__(self, values, placement, ndim=None): @@ -322,14 +321,6 @@ def shape(self): def dtype(self): return self.values.dtype - @property - def ftype(self) -> str: - if getattr(self.values, "_pandas_ftype", False): - dtype = self.dtype.subtype - else: - dtype = self.dtype - return f"{dtype}:{self._ftype}" - def merge(self, other): return _merge_blocks([self, other]) @@ -1956,10 +1947,6 @@ def where( return [self.make_block_same_class(result, placement=self.mgr_locs)] - @property - def _ftype(self): - return getattr(self.values, "_pandas_ftype", Block._ftype) - def _unstack(self, unstacker_func, new_columns, n_rows, fill_value): # ExtensionArray-safe unstack. # We override ObjectBlock._unstack, which unstacks directly on the diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index ada065bc304ff..66e96af05eb71 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -676,8 +676,8 @@ def is_consolidated(self) -> bool: return self._is_consolidated def _consolidate_check(self) -> None: - ftypes = [blk.ftype for blk in self.blocks if blk._can_consolidate] - self._is_consolidated = len(ftypes) == len(set(ftypes)) + dtypes = [blk.dtype for blk in self.blocks if blk._can_consolidate] + self._is_consolidated = len(dtypes) == len(set(dtypes)) self._known_consolidated = True @property