Skip to content

BUG: allow DataFrame[mask]=value with mixed non-numeric dtypes #53291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.1.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ Interval
Indexing
^^^^^^^^
- Bug in :meth:`DataFrame.__setitem__` losing dtype when setting a :class:`DataFrame` into duplicated columns (:issue:`53143`)
- Bug in :meth:`DataFrame.__setitem__` with a boolean mask and :meth:`DataFrame.putmask` with mixed non-numeric dtypes and a value other than ``NaN`` incorrectly raising ``TypeError`` (:issue:`53291`)
-

Missing
Expand Down
1 change: 0 additions & 1 deletion pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -4074,7 +4074,6 @@ def _setitem_frame(self, key, value):
"Must pass DataFrame or 2-d ndarray with boolean values only"
)

self._check_inplace_setting(value)
self._check_setitem_copy()
self._where(-key, value, inplace=True)

Expand Down
17 changes: 0 additions & 17 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,6 @@
is_bool_dtype,
is_dict_like,
is_extension_array_dtype,
is_float,
is_list_like,
is_number,
is_numeric_dtype,
Expand Down Expand Up @@ -6201,21 +6200,6 @@ def _is_mixed_type(self) -> bool_t:

return self.dtypes.nunique() > 1

@final
def _check_inplace_setting(self, value) -> bool_t:
"""check whether we allow in-place setting with this type of value"""
if self._is_mixed_type and not self._mgr.is_numeric_mixed_type:
# allow an actual np.nan through
if (is_float(value) and np.isnan(value)) or value is lib.no_default:
return True

raise TypeError(
"Cannot do inplace boolean setting on "
"mixed-types with a non np.nan value"
)

return True

@final
def _get_numeric_data(self) -> Self:
return self._constructor(self._mgr.get_numeric_data()).__finalize__(self)
Expand Down Expand Up @@ -10036,7 +10020,6 @@ def _where(
# we may have different type blocks come out of putmask, so
# reconstruct the block manager

self._check_inplace_setting(other)
new_data = self._mgr.putmask(mask=cond, new=other, align=align)
result = self._constructor(new_data)
return self._update_inplace(result)
Expand Down
4 changes: 0 additions & 4 deletions pandas/core/internals/array_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -443,10 +443,6 @@ def to_native_types(self, **kwargs) -> Self:
def is_mixed_type(self) -> bool:
return True

@property
def is_numeric_mixed_type(self) -> bool:
return all(is_numeric_dtype(t) for t in self.get_dtypes())

@property
def any_extension_types(self) -> bool:
"""Whether any of the blocks in this manager are extension blocks"""
Expand Down
4 changes: 0 additions & 4 deletions pandas/core/internals/managers.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,10 +514,6 @@ def to_native_types(self, **kwargs) -> Self:
"""
return self.apply("to_native_types", **kwargs)

@property
def is_numeric_mixed_type(self) -> bool:
return all(block.is_numeric for block in self.blocks)

@property
def any_extension_types(self) -> bool:
"""Whether any of the blocks in this manager are extension blocks"""
Expand Down
19 changes: 16 additions & 3 deletions pandas/tests/frame/indexing/test_where.py
Original file line number Diff line number Diff line change
Expand Up @@ -401,10 +401,23 @@ def test_where_none(self):
{"A": np.nan, "B": "Test", "C": np.nan},
]
)
msg = "boolean setting on mixed-type"

with pytest.raises(TypeError, match=msg):
df.where(~isna(df), None, inplace=True)
orig = df.copy()

mask = ~isna(df)
df.where(mask, None, inplace=True)
expected = DataFrame(
{
"A": [1.0, np.nan],
"B": [None, "Test"],
"C": ["Test", None],
}
)
tm.assert_frame_equal(df, expected)

df = orig.copy()
df[~mask] = None
tm.assert_frame_equal(df, expected)

def test_where_empty_df_and_empty_cond_having_non_bool_dtypes(self):
# see gh-21947
Expand Down