Skip to content

BUG: IntervalIndex.putmask with datetimelike dtypes #37968

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Nov 20, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
677bd87
CLN: Index.putmask
jbrockmendel Nov 2, 2020
8bd0d9a
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 2, 2020
561808f
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 3, 2020
9daccd7
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 3, 2020
46ed418
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 4, 2020
2c254af
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 6, 2020
3695efc
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 7, 2020
de8051b
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 9, 2020
c45843f
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 9, 2020
9975839
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 11, 2020
be11215
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 11, 2020
0196ebf
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 13, 2020
1f2f677
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 13, 2020
58a0f31
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 15, 2020
76c01d9
Merge branch 'master' of https://github.com/pandas-dev/pandas into bu…
jbrockmendel Nov 19, 2020
ddf5eef
BUG: IntervalIndex.putmask
jbrockmendel Nov 20, 2020
4a0d918
whatsnew, more tests
jbrockmendel Nov 20, 2020
9dc8666
update GH refs
jbrockmendel Nov 20, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/source/whatsnew/v1.2.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,7 @@ Interval

- Bug in :meth:`DataFrame.replace` and :meth:`Series.replace` where :class:`Interval` dtypes would be converted to object dtypes (:issue:`34871`)
- Bug in :meth:`IntervalIndex.take` with negative indices and ``fill_value=None`` (:issue:`37330`)
-
- Bug in :meth:`IntervalIndex.putmask` with datetime-like dtype incorrectly casting to object dtype (:issue:`37968`)
-

Indexing
Expand Down
21 changes: 21 additions & 0 deletions pandas/core/arrays/_mixins.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,3 +300,24 @@ def __repr__(self) -> str:
data = ",\n".join(lines)
class_name = f"<{type(self).__name__}>"
return f"{class_name}\n[\n{data}\n]\nShape: {self.shape}, dtype: {self.dtype}"

# ------------------------------------------------------------------------
# __array_function__ methods

def putmask(self, mask, value):
"""
Analogue to np.putmask(self, mask, value)

Parameters
----------
mask : np.ndarray[bool]
value : scalar or listlike

Raises
------
TypeError
If value cannot be cast to self.dtype.
"""
value = self._validate_setitem_value(value)

np.putmask(self._ndarray, mask, value)
7 changes: 4 additions & 3 deletions pandas/core/indexes/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -4344,18 +4344,19 @@ def putmask(self, mask, value):
numpy.ndarray.putmask : Changes elements of an array
based on conditional and input values.
"""
values = self.values.copy()
values = self._values.copy()
try:
converted = self._validate_fill_value(value)
np.putmask(values, mask, converted)
return self._shallow_copy(values)
except (ValueError, TypeError) as err:
if is_object_dtype(self):
raise err

# coerces to object
return self.astype(object).putmask(mask, value)

np.putmask(values, mask, converted)
return self._shallow_copy(values)

def equals(self, other: object) -> bool:
"""
Determine if two Index object are equal.
Expand Down
8 changes: 3 additions & 5 deletions pandas/core/indexes/extension.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,15 +359,13 @@ def insert(self, loc: int, item):
return type(self)._simple_new(new_arr, name=self.name)

def putmask(self, mask, value):
res_values = self._data.copy()
try:
value = self._data._validate_setitem_value(value)
res_values.putmask(mask, value)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

self._data is defined as _data: ExtensionArray

Is ExtensionIndex only for ndarray backed ExtensionArray? or maybe will need catch AttributeError also eventually?

see also #35032 (comment) (sort of related).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah my bad for not checking the code thoroughly before commenting and just looking at the diff.

I wasn't aware that you recently added a NDArrayBackedExtensionIndex.

Doesn't this add more complexity? what is the motivation for this?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't this add more complexity? what is the motivation for this?

Its just taking methods that are duplicated in CategoricalIndex+DatetimeIndexOpsMixin and putting them in one place. I guess you could say that "adds complexity" bc there is an additional step in the inheritance chain?

except (TypeError, ValueError):
return self.astype(object).putmask(mask, value)

new_values = self._data._ndarray.copy()
np.putmask(new_values, mask, value)
new_arr = self._data._from_backing_data(new_values)
return type(self)._simple_new(new_arr, name=self.name)
return type(self)._simple_new(res_values, name=self.name)

def _wrap_joined_index(self: _T, joined: np.ndarray, other: _T) -> _T:
name = get_op_result_name(self, other)
Expand Down
16 changes: 16 additions & 0 deletions pandas/core/indexes/interval.py
Original file line number Diff line number Diff line change
Expand Up @@ -858,6 +858,22 @@ def mid(self):
def length(self):
return Index(self._data.length, copy=False)

def putmask(self, mask, value):
arr = self._data.copy()
try:
value_left, value_right = arr._validate_setitem_value(value)
except (ValueError, TypeError):
return self.astype(object).putmask(mask, value)

if isinstance(self._data._left, np.ndarray):
np.putmask(arr._left, mask, value_left)
np.putmask(arr._right, mask, value_right)
else:
# TODO: special case not needed with __array_function__
arr._left.putmask(mask, value_left)
arr._right.putmask(mask, value_right)
return type(self)._simple_new(arr, name=self.name)

@Appender(Index.where.__doc__)
def where(self, cond, other=None):
if other is None:
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/indexes/numeric.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ def _validate_fill_value(self, value):
# force conversion to object
# so we don't lose the bools
raise TypeError
if isinstance(value, str):
elif isinstance(value, str) or lib.is_complex(value):
raise TypeError

return value
Expand Down
24 changes: 24 additions & 0 deletions pandas/tests/indexes/interval/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,30 @@ def test_where(self, closed, klass):
result = idx.where(klass(cond))
tm.assert_index_equal(result, expected)

@pytest.mark.parametrize("tz", ["US/Pacific", None])
def test_putmask_dt64(self, tz):
# GH#37968
dti = date_range("2016-01-01", periods=9, tz=tz)
idx = IntervalIndex.from_breaks(dti)
mask = np.zeros(idx.shape, dtype=bool)
mask[0:3] = True

result = idx.putmask(mask, idx[-1])
expected = IntervalIndex([idx[-1]] * 3 + list(idx[3:]))
tm.assert_index_equal(result, expected)

def test_putmask_td64(self):
# GH#37968
dti = date_range("2016-01-01", periods=9)
tdi = dti - dti[0]
idx = IntervalIndex.from_breaks(tdi)
mask = np.zeros(idx.shape, dtype=bool)
mask[0:3] = True

result = idx.putmask(mask, idx[-1])
expected = IntervalIndex([idx[-1]] * 3 + list(idx[3:]))
tm.assert_index_equal(result, expected)

def test_getitem_2d_deprecated(self):
# GH#30588 multi-dim indexing is deprecated, but raising is also acceptable
idx = self.create_index()
Expand Down