Skip to content

Commit d717ad8

Browse files
jbrockmendeljreback
authored andcommitted
CLN/REF: stop allowing iNaT in DatetimeBlock (#27428)
1 parent 05b3b96 commit d717ad8

File tree

4 files changed

+28
-34
lines changed

4 files changed

+28
-34
lines changed

pandas/core/internals/blocks.py

+12-23
Original file line numberDiff line numberDiff line change
@@ -2184,7 +2184,7 @@ def _holder(self):
21842184

21852185
@property
21862186
def fill_value(self):
2187-
return tslibs.iNaT
2187+
return np.datetime64("NaT", "ns")
21882188

21892189
def get_values(self, dtype=None):
21902190
"""
@@ -2266,14 +2266,9 @@ def _can_hold_element(self, element):
22662266
if self.is_datetimetz:
22672267
return tz_compare(element.tzinfo, self.dtype.tz)
22682268
return element.tzinfo is None
2269-
elif is_integer(element):
2270-
return element == tslibs.iNaT
22712269

22722270
return is_valid_nat_for_dtype(element, self.dtype)
22732271

2274-
def _coerce_values(self, values):
2275-
return values.view("i8")
2276-
22772272
def _try_coerce_args(self, other):
22782273
"""
22792274
Coerce other to dtype 'i8'. NaN and NaT convert to
@@ -2290,16 +2285,15 @@ def _try_coerce_args(self, other):
22902285
base-type other
22912286
"""
22922287
if is_valid_nat_for_dtype(other, self.dtype):
2293-
other = tslibs.iNaT
2294-
elif is_integer(other) and other == tslibs.iNaT:
2295-
pass
2288+
other = np.datetime64("NaT", "ns")
22962289
elif isinstance(other, (datetime, np.datetime64, date)):
22972290
other = self._box_func(other)
22982291
if getattr(other, "tz") is not None:
22992292
raise TypeError("cannot coerce a Timestamp with a tz on a naive Block")
2300-
other = other.asm8.view("i8")
2293+
other = other.asm8
23012294
elif hasattr(other, "dtype") and is_datetime64_dtype(other):
2302-
other = other.astype("i8", copy=False).view("i8")
2295+
# TODO: can we get here with non-nano?
2296+
pass
23032297
else:
23042298
# coercion issues
23052299
# let higher levels handle
@@ -2458,8 +2452,7 @@ def _slice(self, slicer):
24582452
return self.values[slicer]
24592453

24602454
def _coerce_values(self, values):
2461-
# asi8 is a view, needs copy
2462-
return _block_shape(values.view("i8"), ndim=self.ndim)
2455+
return _block_shape(values, ndim=self.ndim)
24632456

24642457
def _try_coerce_args(self, other):
24652458
"""
@@ -2484,21 +2477,17 @@ def _try_coerce_args(self, other):
24842477
other = self._holder(other, dtype=self.dtype)
24852478

24862479
elif is_valid_nat_for_dtype(other, self.dtype):
2487-
other = tslibs.iNaT
2488-
elif is_integer(other) and other == tslibs.iNaT:
2489-
pass
2480+
other = np.datetime64("NaT", "ns")
24902481
elif isinstance(other, self._holder):
2491-
if other.tz != self.values.tz:
2482+
if not tz_compare(other.tz, self.values.tz):
24922483
raise ValueError("incompatible or non tz-aware value")
2493-
other = _block_shape(other.asi8, ndim=self.ndim)
2484+
24942485
elif isinstance(other, (np.datetime64, datetime, date)):
24952486
other = tslibs.Timestamp(other)
2496-
tz = getattr(other, "tz", None)
24972487

24982488
# test we can have an equal time zone
2499-
if tz is None or str(tz) != str(self.values.tz):
2489+
if not tz_compare(other.tz, self.values.tz):
25002490
raise ValueError("incompatible or non tz-aware value")
2501-
other = other.value
25022491
else:
25032492
raise TypeError(other)
25042493

@@ -2654,8 +2643,8 @@ def fillna(self, value, **kwargs):
26542643

26552644
def _try_coerce_args(self, other):
26562645
"""
2657-
Coerce values and other to int64, with null values converted to
2658-
iNaT. values is always ndarray-like, other may not be
2646+
Coerce values and other to datetime64[ns], with null values
2647+
converted to datetime64("NaT", "ns").
26592648
26602649
Parameters
26612650
----------

pandas/core/nanops.py

+10-8
Original file line numberDiff line numberDiff line change
@@ -1360,14 +1360,6 @@ def _nanpercentile_1d(values, mask, q, na_value, interpolation):
13601360
quantiles : scalar or array
13611361
"""
13621362
# mask is Union[ExtensionArray, ndarray]
1363-
if values.dtype.kind == "m":
1364-
# need to cast to integer to avoid rounding errors in numpy
1365-
result = _nanpercentile_1d(values.view("i8"), mask, q, na_value, interpolation)
1366-
1367-
# Note: we have to do do `astype` and not view because in general we
1368-
# have float result at this point, not i8
1369-
return result.astype(values.dtype)
1370-
13711363
values = values[~mask]
13721364

13731365
if len(values) == 0:
@@ -1399,6 +1391,16 @@ def nanpercentile(values, q, axis, na_value, mask, ndim, interpolation):
13991391
-------
14001392
quantiles : scalar or array
14011393
"""
1394+
if values.dtype.kind in ["m", "M"]:
1395+
# need to cast to integer to avoid rounding errors in numpy
1396+
result = nanpercentile(
1397+
values.view("i8"), q, axis, na_value.view("i8"), mask, ndim, interpolation
1398+
)
1399+
1400+
# Note: we have to do do `astype` and not view because in general we
1401+
# have float result at this point, not i8
1402+
return result.astype(values.dtype)
1403+
14021404
if not lib.is_scalar(mask) and mask.any():
14031405
if ndim == 1:
14041406
return _nanpercentile_1d(

pandas/tests/frame/test_indexing.py

+5-2
Original file line numberDiff line numberDiff line change
@@ -1150,6 +1150,7 @@ def test_fancy_index_int_labels_exceptions(self, float_frame):
11501150
with pytest.raises(KeyError, match=msg):
11511151
float_frame.ix[:, ["E"]] = 1
11521152

1153+
# FIXME: don't leave commented-out
11531154
# partial setting now allows this GH2578
11541155
# pytest.raises(KeyError, float_frame.ix.__setitem__,
11551156
# (slice(None, None), 'E'), 1)
@@ -1676,9 +1677,11 @@ def test_setitem_single_column_mixed_datetime(self):
16761677
)
16771678
assert_series_equal(result, expected)
16781679

1679-
# set an allowable datetime64 type
1680+
# GH#16674 iNaT is treated as an integer when given by the user
16801681
df.loc["b", "timestamp"] = iNaT
1681-
assert isna(df.loc["b", "timestamp"])
1682+
assert not isna(df.loc["b", "timestamp"])
1683+
assert df["timestamp"].dtype == np.object_
1684+
assert df.loc["b", "timestamp"] == iNaT
16821685

16831686
# allow this syntax
16841687
df.loc["c", "timestamp"] = np.nan

pandas/tests/internals/test_internals.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ def test_try_coerce_arg(self):
338338
vals = (np.datetime64("2010-10-10"), datetime(2010, 10, 10), date(2010, 10, 10))
339339
for val in vals:
340340
coerced = block._try_coerce_args(val)
341-
assert np.int64 == type(coerced)
341+
assert np.datetime64 == type(coerced)
342342
assert pd.Timestamp("2010-10-10") == pd.Timestamp(coerced)
343343

344344

0 commit comments

Comments
 (0)