Skip to content

REF: dont consolidate in is_mixed_type #36873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 8, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -630,7 +630,6 @@ def _is_homogeneous_type(self) -> bool:
if self._mgr.any_extension_types:
return len({block.dtype for block in self._mgr.blocks}) == 1
else:
# Note: consolidates inplace
return not self._is_mixed_type

@property
Expand Down
14 changes: 10 additions & 4 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -5472,8 +5472,15 @@ def _consolidate(self, inplace: bool_t = False):

@property
def _is_mixed_type(self) -> bool_t:
f = lambda: self._mgr.is_mixed_type
return self._protect_consolidate(f)
if len(self._mgr.blocks) == 1:
return False

if self._mgr.any_extension_types:
# Even if they have the same dtype, we cant consolidate them,
# so we pretend this is "mixed'"
return True

return self.dtypes.nunique() > 1

def _check_inplace_setting(self, value) -> bool_t:
""" check whether we allow in-place setting with this type of value """
Expand Down Expand Up @@ -6253,8 +6260,7 @@ def fillna(
axis = self._get_axis_number(axis)

if value is None:

if self._is_mixed_type and axis == 1:
if len(self._mgr.blocks) > 1 and axis == 1:
if inplace:
raise NotImplementedError()
result = self.T.fillna(method=method, limit=limit).T
Expand Down
3 changes: 1 addition & 2 deletions pandas/core/indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1547,8 +1547,7 @@ def _setitem_with_indexer(self, indexer, value):
info_axis = self.obj._info_axis_number

# maybe partial set
# _is_mixed_type has the side effect of consolidating in-place
take_split_path = self.obj._is_mixed_type
take_split_path = len(self.obj._mgr.blocks) > 1

# if there is only one block/type, still have to take split path
# unless the block is one-dimensional or it can hold the value
Expand Down
6 changes: 0 additions & 6 deletions pandas/core/internals/managers.py
Original file line number Diff line number Diff line change
Expand Up @@ -651,12 +651,6 @@ def _consolidate_check(self) -> None:
self._is_consolidated = len(dtypes) == len(set(dtypes))
self._known_consolidated = True

@property
def is_mixed_type(self) -> bool:
# Warning, consolidation needs to get checked upstairs
self._consolidate_inplace()
return len(self.blocks) > 1

@property
def is_numeric_mixed_type(self) -> bool:
return all(block.is_numeric for block in self.blocks)
Expand Down
12 changes: 0 additions & 12 deletions pandas/tests/frame/test_dtypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,18 +230,6 @@ def test_constructor_list_str_na(self, string_dtype):
def test_is_homogeneous_type(self, data, expected):
assert data._is_homogeneous_type is expected

def test_is_homogeneous_type_clears_cache(self):
ser = pd.Series([1, 2, 3])
df = ser.to_frame("A")
df["B"] = ser

assert len(df._mgr.blocks) == 2

a = df["B"] # caches lookup
df._is_homogeneous_type # _should_ clear cache
assert len(df._mgr.blocks) == 1
assert df["B"] is not a

def test_asarray_homogenous(self):
df = pd.DataFrame({"A": pd.Categorical([1, 2]), "B": pd.Categorical([1, 2])})
result = np.asarray(df)
Expand Down
7 changes: 0 additions & 7 deletions pandas/tests/internals/test_internals.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,13 +272,6 @@ def test_attrs(self):
assert mgr.nblocks == 2
assert len(mgr) == 6

def test_is_mixed_dtype(self):
assert not create_mgr("a,b:f8").is_mixed_type
assert not create_mgr("a:f8-1; b:f8-2").is_mixed_type

assert create_mgr("a,b:f8; c,d: f4").is_mixed_type
assert create_mgr("a,b:f8; c,d: object").is_mixed_type

def test_duplicate_ref_loc_failure(self):
tmp_mgr = create_mgr("a:bool; a: f8")

Expand Down