Skip to content

REF: use DatetimeArray._validate_setitem_value for DatetimeBlock._can_hold_element #38651

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 23, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pandas/core/arrays/datetimelike.py
Original file line number Diff line number Diff line change
Expand Up @@ -601,6 +601,10 @@ def _validate_listlike(self, value, allow_object: bool = False):
if isinstance(value, type(self)):
return value

if isinstance(value, list) and len(value) == 0:
# We treat empty list as our own dtype.
return type(self)._from_sequence([], dtype=self.dtype)

# Do type inference if necessary up front
# e.g. we passed PeriodIndex.values and got an ndarray of Periods
value = array(value)
Expand Down
38 changes: 10 additions & 28 deletions pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime, timedelta
from datetime import timedelta
import inspect
import re
from typing import TYPE_CHECKING, Any, List, Optional, Type, Union, cast
Expand All @@ -18,7 +18,6 @@
)
from pandas._libs.internals import BlockPlacement
from pandas._libs.tslibs import conversion
from pandas._libs.tslibs.timezones import tz_compare
from pandas._typing import ArrayLike, DtypeObj, Scalar, Shape
from pandas.util._validators import validate_bool_kwarg

Expand Down Expand Up @@ -2190,6 +2189,15 @@ def to_native_types(self, na_rep="NaT", **kwargs):
result = arr._format_native_types(na_rep=na_rep, **kwargs)
return self.make_block(result)

def _can_hold_element(self, element: Any) -> bool:
arr = self.array_values()

try:
arr._validate_setitem_value(element)
return True
except (TypeError, ValueError):
return False


class DatetimeBlock(DatetimeLikeBlockMixin):
__slots__ = ()
Expand Down Expand Up @@ -2222,32 +2230,6 @@ def _maybe_coerce_values(self, values):
assert isinstance(values, np.ndarray), type(values)
return values

def _can_hold_element(self, element: Any) -> bool:
tipo = maybe_infer_dtype_type(element)
if tipo is not None:
if isinstance(element, list) and len(element) == 0:
# Following DatetimeArray._validate_setitem_value
# convention, we treat this as object-dtype
# (even though tipo is float64)
return True

elif self.is_datetimetz:
# require exact match, since non-nano does not exist
return is_dtype_equal(tipo, self.dtype) or is_valid_nat_for_dtype(
element, self.dtype
)

# GH#27419 if we get a non-nano datetime64 object
return is_datetime64_dtype(tipo)
elif element is NaT:
return True
elif isinstance(element, datetime):
if self.is_datetimetz:
return tz_compare(element.tzinfo, self.dtype.tz)
return element.tzinfo is None

return is_valid_nat_for_dtype(element, self.dtype)

def set_inplace(self, locs, values):
"""
See Block.set.__doc__
Expand Down
52 changes: 51 additions & 1 deletion pandas/tests/indexing/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from pandas.core.dtypes.common import is_float_dtype, is_integer_dtype

import pandas as pd
from pandas import DataFrame, Index, NaT, Series
from pandas import DataFrame, Index, NaT, Series, date_range
import pandas._testing as tm
from pandas.core.indexing import maybe_numeric_slice, non_reducing_slice
from pandas.tests.indexing.common import _mklbl
Expand Down Expand Up @@ -966,6 +966,56 @@ def test_none_coercion_mixed_dtypes(self):
tm.assert_frame_equal(start_dataframe, exp)


class TestDatetimelikeCoercion:
@pytest.mark.parametrize("indexer", [setitem, loc, iloc])
def test_setitem_dt64_string_scalar(self, tz_naive_fixture, indexer):
# dispatching _can_hold_element to underling DatetimeArray
# TODO(EA2D) use tz_naive_fixture once DatetimeBlock is backed by DTA
tz = tz_naive_fixture

dti = date_range("2016-01-01", periods=3, tz=tz)
ser = Series(dti)

values = ser._values

indexer(ser)[0] = "2018-01-01"

if tz is None:
# TODO(EA2D): we can make this no-copy in tz-naive case too
assert ser.dtype == dti.dtype
else:
assert ser._values is values

@pytest.mark.parametrize("box", [list, np.array, pd.array])
@pytest.mark.parametrize(
"key", [[0, 1], slice(0, 2), np.array([True, True, False])]
)
@pytest.mark.parametrize("indexer", [setitem, loc, iloc])
def test_setitem_dt64_string_values(self, tz_naive_fixture, indexer, key, box):
# dispatching _can_hold_element to underling DatetimeArray
# TODO(EA2D) use tz_naive_fixture once DatetimeBlock is backed by DTA
tz = tz_naive_fixture

if isinstance(key, slice) and indexer is loc:
key = slice(0, 1)

dti = date_range("2016-01-01", periods=3, tz=tz)
ser = Series(dti)

values = ser._values

newvals = box(["2019-01-01", "2010-01-02"])
values._validate_setitem_value(newvals)

indexer(ser)[key] = newvals

if tz is None:
# TODO(EA2D): we can make this no-copy in tz-naive case too
assert ser.dtype == dti.dtype
else:
assert ser._values is values


def test_extension_array_cross_section():
# A cross-section of a homogeneous EA should be an EA
df = DataFrame(
Expand Down