From 129e945d3e3213aa13431c7d90ffb8681a128ff5 Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 26 Nov 2020 18:47:28 -0800 Subject: [PATCH 1/3] REF: Use _validate_setitem_value for _can_hold_element --- pandas/core/arrays/datetimelike.py | 4 ++++ pandas/core/internals/blocks.py | 38 ++++++++---------------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 79ecf8620c70c..9eba4f85f23f5 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -601,6 +601,10 @@ def _validate_listlike(self, value, allow_object: bool = False): if isinstance(value, type(self)): return value + if isinstance(value, list) and len(value) == 0: + # We treat empty list as our own dtype. + return type(self)._from_sequence([], dtype=self.dtype) + # Do type inference if necessary up front # e.g. we passed PeriodIndex.values and got an ndarray of Periods value = array(value) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 6500cfcd9ea5a..8752224356f61 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import timedelta import inspect import re from typing import TYPE_CHECKING, Any, List, Optional, Type, Union, cast @@ -18,7 +18,6 @@ ) from pandas._libs.internals import BlockPlacement from pandas._libs.tslibs import conversion -from pandas._libs.tslibs.timezones import tz_compare from pandas._typing import ArrayLike, DtypeObj, Scalar, Shape from pandas.util._validators import validate_bool_kwarg @@ -2190,6 +2189,15 @@ def to_native_types(self, na_rep="NaT", **kwargs): result = arr._format_native_types(na_rep=na_rep, **kwargs) return self.make_block(result) + def _can_hold_element(self, element: Any) -> bool: + arr = self.array_values() + + try: + arr._validate_setitem_value(element) + return True + except (TypeError, ValueError): + return False + class DatetimeBlock(DatetimeLikeBlockMixin): __slots__ = () @@ -2222,32 +2230,6 @@ def _maybe_coerce_values(self, values): assert isinstance(values, np.ndarray), type(values) return values - def _can_hold_element(self, element: Any) -> bool: - tipo = maybe_infer_dtype_type(element) - if tipo is not None: - if isinstance(element, list) and len(element) == 0: - # Following DatetimeArray._validate_setitem_value - # convention, we treat this as object-dtype - # (even though tipo is float64) - return True - - elif self.is_datetimetz: - # require exact match, since non-nano does not exist - return is_dtype_equal(tipo, self.dtype) or is_valid_nat_for_dtype( - element, self.dtype - ) - - # GH#27419 if we get a non-nano datetime64 object - return is_datetime64_dtype(tipo) - elif element is NaT: - return True - elif isinstance(element, datetime): - if self.is_datetimetz: - return tz_compare(element.tzinfo, self.dtype.tz) - return element.tzinfo is None - - return is_valid_nat_for_dtype(element, self.dtype) - def set_inplace(self, locs, values): """ See Block.set.__doc__ From 1473e1751400a3aacc0676f61ffa9f2acd0f3850 Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 22 Dec 2020 18:14:07 -0800 Subject: [PATCH 2/3] tests --- pandas/tests/indexing/test_indexing.py | 39 +++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 5cd429837a127..12fbe03097646 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -10,7 +10,7 @@ from pandas.core.dtypes.common import is_float_dtype, is_integer_dtype import pandas as pd -from pandas import DataFrame, Index, NaT, Series +from pandas import DataFrame, Index, NaT, Series, date_range import pandas._testing as tm from pandas.core.indexing import maybe_numeric_slice, non_reducing_slice from pandas.tests.indexing.common import _mklbl @@ -966,6 +966,43 @@ def test_none_coercion_mixed_dtypes(self): tm.assert_frame_equal(start_dataframe, exp) +class TestDatetimelikeCoercion: + @pytest.mark.parametrize("indexer", [setitem, loc, iloc]) + def test_setitem_dt64_string_scalar(self, tz_aware_fixture, indexer): + # dispatching _can_hold_element to underling DatetimeArray + # TODO(EA2D) use tz_naive_fixture once DatetimeBlock is backed by DTA + dti = date_range("2016-01-01", periods=3, tz=tz_aware_fixture) + ser = Series(dti) + + values = ser._values + + indexer(ser)[0] = "2018-01-01" + assert ser.dtype == dti.dtype + assert ser._values is values + + @pytest.mark.parametrize("box", [list, np.array, pd.array]) + @pytest.mark.parametrize( + "key", [[0, 1], slice(0, 2), np.array([True, True, False])] + ) + @pytest.mark.parametrize("indexer", [setitem, loc, iloc]) + def test_setitem_dt64_string_values(self, tz_aware_fixture, indexer, key, box): + # dispatching _can_hold_element to underling DatetimeArray + # TODO(EA2D) use tz_naive_fixture once DatetimeBlock is backed by DTA + if isinstance(key, slice) and indexer is loc: + key = slice(0, 1) + + dti = date_range("2016-01-01", periods=3, tz=tz_aware_fixture) + ser = Series(dti) + + values = ser._values + + newvals = box(["2019-01-01", "2010-01-02"]) + values._validate_setitem_value(newvals) + + indexer(ser)[key] = newvals + assert ser._values is values + + def test_extension_array_cross_section(): # A cross-section of a homogeneous EA should be an EA df = DataFrame( From 465639d6d40a21a2c43a0a4a1f3608b903bf0a67 Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 22 Dec 2020 18:17:37 -0800 Subject: [PATCH 3/3] test tznaive case too --- pandas/tests/indexing/test_indexing.py | 27 +++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 12fbe03097646..05d9d1a9bd74f 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -968,30 +968,38 @@ def test_none_coercion_mixed_dtypes(self): class TestDatetimelikeCoercion: @pytest.mark.parametrize("indexer", [setitem, loc, iloc]) - def test_setitem_dt64_string_scalar(self, tz_aware_fixture, indexer): + def test_setitem_dt64_string_scalar(self, tz_naive_fixture, indexer): # dispatching _can_hold_element to underling DatetimeArray # TODO(EA2D) use tz_naive_fixture once DatetimeBlock is backed by DTA - dti = date_range("2016-01-01", periods=3, tz=tz_aware_fixture) + tz = tz_naive_fixture + + dti = date_range("2016-01-01", periods=3, tz=tz) ser = Series(dti) values = ser._values indexer(ser)[0] = "2018-01-01" - assert ser.dtype == dti.dtype - assert ser._values is values + + if tz is None: + # TODO(EA2D): we can make this no-copy in tz-naive case too + assert ser.dtype == dti.dtype + else: + assert ser._values is values @pytest.mark.parametrize("box", [list, np.array, pd.array]) @pytest.mark.parametrize( "key", [[0, 1], slice(0, 2), np.array([True, True, False])] ) @pytest.mark.parametrize("indexer", [setitem, loc, iloc]) - def test_setitem_dt64_string_values(self, tz_aware_fixture, indexer, key, box): + def test_setitem_dt64_string_values(self, tz_naive_fixture, indexer, key, box): # dispatching _can_hold_element to underling DatetimeArray # TODO(EA2D) use tz_naive_fixture once DatetimeBlock is backed by DTA + tz = tz_naive_fixture + if isinstance(key, slice) and indexer is loc: key = slice(0, 1) - dti = date_range("2016-01-01", periods=3, tz=tz_aware_fixture) + dti = date_range("2016-01-01", periods=3, tz=tz) ser = Series(dti) values = ser._values @@ -1000,7 +1008,12 @@ def test_setitem_dt64_string_values(self, tz_aware_fixture, indexer, key, box): values._validate_setitem_value(newvals) indexer(ser)[key] = newvals - assert ser._values is values + + if tz is None: + # TODO(EA2D): we can make this no-copy in tz-naive case too + assert ser.dtype == dti.dtype + else: + assert ser._values is values def test_extension_array_cross_section():