Skip to content

Fix _can_hold_element for datetimelike blocks #27347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Jul 15, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
0b5adeb
make _can_hold_element more accurate
jbrockmendel Jul 11, 2019
9a79b37
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 11, 2019
e1f8703
Correctly accept None, np.nan, timedelta64(NaT)
jbrockmendel Jul 11, 2019
5c6ccd3
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 11, 2019
96e666f
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 11, 2019
3663e22
revert m8 change
jbrockmendel Jul 11, 2019
5a63cf4
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 11, 2019
36c3b58
use is_valid_nat_for_dtype
jbrockmendel Jul 11, 2019
e0f76a6
test_roundtrip_thru_setitem fix
jbrockmendel Jul 11, 2019
c26cd12
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 12, 2019
06b54ef
use is_valid_nat_for_dtype
jbrockmendel Jul 12, 2019
7a54804
allow iNaT through
jbrockmendel Jul 12, 2019
6ae4ee1
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 12, 2019
07be36c
scalar check
jbrockmendel Jul 12, 2019
f4d381f
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 13, 2019
1b1d4d8
avoid np deprecation warnings that raise in npdev
jbrockmendel Jul 13, 2019
178dde6
implement is_integer_strict
jbrockmendel Jul 15, 2019
060998e
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 15, 2019
a126c3b
revert is_integer_strict
jbrockmendel Jul 15, 2019
aec6a6c
Merge branch 'master' of https://github.com/pandas-dev/pandas into ca…
jbrockmendel Jul 15, 2019
2f3a0ed
update post is_integer fix
jbrockmendel Jul 15, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pandas/core/dtypes/missing.py
Original file line number Diff line number Diff line change
Expand Up @@ -574,7 +574,7 @@ def is_valid_nat_for_dtype(obj, dtype):
-------
bool
"""
if not isna(obj):
if not lib.is_scalar(obj) or not isna(obj):
return False
if dtype.kind == "M":
return not isinstance(obj, np.timedelta64)
Expand Down
42 changes: 33 additions & 9 deletions pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@

from pandas._libs import NaT, lib, tslib, tslibs
import pandas._libs.internals as libinternals
from pandas._libs.tslibs import Timedelta, conversion, is_null_datetimelike
from pandas._libs.tslibs import Timedelta, conversion
from pandas._libs.tslibs.timezones import tz_compare
from pandas.util._validators import validate_bool_kwarg

from pandas.core.dtypes.cast import (
Expand Down Expand Up @@ -60,7 +61,13 @@
ABCPandasArray,
ABCSeries,
)
from pandas.core.dtypes.missing import _isna_compat, array_equivalent, isna, notna
from pandas.core.dtypes.missing import (
_isna_compat,
array_equivalent,
is_valid_nat_for_dtype,
isna,
notna,
)

import pandas.core.algorithms as algos
from pandas.core.arrays import (
Expand Down Expand Up @@ -2248,14 +2255,17 @@ def _astype(self, dtype, **kwargs):
def _can_hold_element(self, element):
tipo = maybe_infer_dtype_type(element)
if tipo is not None:
return tipo == _NS_DTYPE or tipo == np.int64
return is_dtype_equal(tipo, self.dtype)
elif element is NaT:
return True
elif isinstance(element, datetime):
if self.is_datetimetz:
return tz_compare(element.tzinfo, self.dtype.tz)
return element.tzinfo is None
elif is_integer(element):
return element == tslibs.iNaT

# TODO: shouldnt we exclude timedelta64("NaT")? See GH#27297
return isna(element)
return is_valid_nat_for_dtype(element, self.dtype)

def _coerce_values(self, values):
return values.view("i8")
Expand All @@ -2275,8 +2285,10 @@ def _try_coerce_args(self, other):
-------
base-type other
"""
if is_null_datetimelike(other):
if is_valid_nat_for_dtype(other, self.dtype):
other = tslibs.iNaT
elif is_integer(other) and other == tslibs.iNaT:
pass
elif isinstance(other, (datetime, np.datetime64, date)):
other = self._box_func(other)
if getattr(other, "tz") is not None:
Expand Down Expand Up @@ -2359,6 +2371,8 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeBlock):
is_datetimetz = True
is_extension = True

_can_hold_element = DatetimeBlock._can_hold_element

@property
def _holder(self):
return DatetimeArray
Expand Down Expand Up @@ -2465,8 +2479,10 @@ def _try_coerce_args(self, other):
# add the tz back
other = self._holder(other, dtype=self.dtype)

elif is_null_datetimelike(other):
elif is_valid_nat_for_dtype(other, self.dtype):
other = tslibs.iNaT
elif is_integer(other) and other == tslibs.iNaT:
pass
elif isinstance(other, self._holder):
if other.tz != self.values.tz:
raise ValueError("incompatible or non tz-aware value")
Expand Down Expand Up @@ -2606,10 +2622,16 @@ def _box_func(self):
def _can_hold_element(self, element):
tipo = maybe_infer_dtype_type(element)
if tipo is not None:
# TODO: remove the np.int64 support once coerce_values and
# _try_coerce_args both coerce to m8[ns] and not i8.
return issubclass(tipo.type, (np.timedelta64, np.int64))
elif element is NaT:
return True
return is_integer(element) or isinstance(element, (timedelta, np.timedelta64))
elif isinstance(element, (timedelta, np.timedelta64)):
return True
elif is_integer(element):
return element == tslibs.iNaT
return is_valid_nat_for_dtype(element, self.dtype)

def fillna(self, value, **kwargs):

Expand Down Expand Up @@ -2645,8 +2667,10 @@ def _try_coerce_args(self, other):
base-type other
"""

if is_null_datetimelike(other):
if is_valid_nat_for_dtype(other, self.dtype):
other = tslibs.iNaT
elif is_integer(other) and other == tslibs.iNaT:
pass
elif isinstance(other, (timedelta, np.timedelta64)):
other = Timedelta(other).value
elif hasattr(other, "dtype") and is_timedelta64_dtype(other):
Expand Down