Merge remote-tracking branch 'upstream/main' into ci/align_azure

Author: mroeschke

doc/source/whatsnew/v1.5.0.rst

@@ -266,6 +266,7 @@ Indexing
 - Bug in :meth:`DataFrame.iloc` where indexing a single row on a :class:`DataFrame` with a single ExtensionDtype column gave a copy instead of a view on the underlying data (:issue:`45241`)
 - Bug in setting a NA value (``None`` or ``np.nan``) into a :class:`Series` with int-based :class:`IntervalDtype` incorrectly casting to object dtype instead of a float-based :class:`IntervalDtype` (:issue:`45568`)
 - Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised insead of casting to a common dtype (:issue:`45070`)
+- Bug in :meth:`Series.__setitem__` when setting incompatible values into a ``PeriodDtype`` or ``IntervalDtype`` :class:`Series` raising when indexing with a boolean mask but coercing when indexing with otherwise-equivalent indexers; these now consistently coerce, along with :meth:`Series.mask` and :meth:`Series.where` (:issue:`45768`)
 - Bug in :meth:`Series.loc.__setitem__` and :meth:`Series.loc.__getitem__` not raising when using multiple keys without using a :class:`MultiIndex` (:issue:`13831`)
 - Bug when setting a value too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`, :issue:`32878`)
 - Bug in :meth:`loc.__setitem__` treating ``range`` keys as positional instead of label-based (:issue:`45479`)

pandas/core/indexes/base.py

@@ -391,7 +391,6 @@ def _outer_indexer(
     _comparables: list[str] = ["name"]
     _attributes: list[str] = ["name"]
     _is_numeric_dtype: bool = False
-    _can_hold_na: bool = True
     _can_hold_strings: bool = True
 
     # Whether this index is a NumericIndex, but not a Int64Index, Float64Index,
@@ -2206,6 +2205,20 @@ def _get_grouper_for_level(self, mapper, *, level=None):
     # --------------------------------------------------------------------
     # Introspection Methods
 
+    @cache_readonly
+    @final
+    def _can_hold_na(self) -> bool:
+        if isinstance(self.dtype, ExtensionDtype):
+            if isinstance(self.dtype, IntervalDtype):
+                # FIXME(GH#45720): this is inaccurate for integer-backed
+                #  IntervalArray, but without it other.categories.take raises
+                #  in IntervalArray._cmp_method
+                return True
+            return self.dtype._can_hold_na
+        if self.dtype.kind in ["i", "u", "b"]:
+            return False
+        return True
+
     @final
     @property
     def is_monotonic(self) -> bool:
@@ -2662,10 +2675,21 @@ def inferred_type(self) -> str_t:
         return lib.infer_dtype(self._values, skipna=False)
 
     @cache_readonly
+    @final
     def _is_all_dates(self) -> bool:
         """
         Whether or not the index values only consist of dates.
         """
+
+        if needs_i8_conversion(self.dtype):
+            return True
+        elif self.dtype != _dtype_obj:
+            # TODO(ExtensionIndex): 3rd party EA might override?
+            # Note: this includes IntervalIndex, even when the left/right
+            #  contain datetime-like objects.
+            return False
+        elif self._is_multi:
+            return False
         return is_datetime_array(ensure_object(self._values))
 
     @cache_readonly
@@ -6159,6 +6183,10 @@ def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
         """
         Can we compare values of the given dtype to our own?
         """
+        if self.dtype.kind == "b":
+            return dtype.kind == "b"
+        elif is_numeric_dtype(self.dtype):
+            return is_numeric_dtype(dtype)
         return True
 
     @final

pandas/core/indexes/datetimelike.py

@@ -100,10 +100,6 @@ class DatetimeIndexOpsMixin(NDArrayBackedExtensionIndex):
         ),
     )
 
-    @property
-    def _is_all_dates(self) -> bool:
-        return True
-
     # ------------------------------------------------------------------------
 
     def equals(self, other: Any) -> bool:
@@ -151,8 +147,6 @@ def __contains__(self, key: Any) -> bool:
             return False
         return True
 
-    _can_hold_na = True
-
     def _convert_tolerance(self, tolerance, target):
         tolerance = np.asarray(to_timedelta(tolerance).to_numpy())
         return super()._convert_tolerance(tolerance, target)

pandas/core/indexes/interval.py

@@ -897,14 +897,6 @@ def _intersection_non_unique(self, other: IntervalIndex) -> IntervalIndex:
 
     # --------------------------------------------------------------------
 
-    @property
-    def _is_all_dates(self) -> bool:
-        """
-        This is False even when left/right contain datetime-like objects,
-        as the check is done on the Interval itself
-        """
-        return False
-
     def _get_engine_target(self) -> np.ndarray:
         # Note: we _could_ use libjoin functions by either casting to object
         #  dtype or constructing tuples (faster than constructing Intervals)

pandas/core/indexes/multi.py

@@ -1841,10 +1841,6 @@ def to_flat_index(self) -> Index:
         """
         return Index(self._values, tupleize_cols=False)
 
-    @property
-    def _is_all_dates(self) -> bool:
-        return False
-
     def is_lexsorted(self) -> bool:
         warnings.warn(
             "MultiIndex.is_lexsorted is deprecated as a public function, "

pandas/core/indexes/numeric.py

@@ -14,7 +14,6 @@
 )
 from pandas._typing import (
     Dtype,
-    DtypeObj,
     npt,
 )
 from pandas.util._decorators import (
@@ -91,14 +90,6 @@ class NumericIndex(Index):
     _can_hold_strings = False
     _is_backward_compat_public_numeric_index: bool = True
 
-    # error: Signature of "_can_hold_na" incompatible with supertype "Index"
-    @cache_readonly
-    def _can_hold_na(self) -> bool:  # type: ignore[override]
-        if is_float_dtype(self.dtype):
-            return True
-        else:
-            return False
-
     _engine_types: dict[np.dtype, type[libindex.IndexEngine]] = {
         np.dtype(np.int8): libindex.Int8Engine,
         np.dtype(np.int16): libindex.Int16Engine,
@@ -268,10 +259,6 @@ def _convert_tolerance(self, tolerance, target):
                 )
         return tolerance
 
-    def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
-        # If we ever have BoolIndex or ComplexIndex, this may need to be tightened
-        return is_numeric_dtype(dtype)
-
     @classmethod
     def _assert_safe_casting(cls, data: np.ndarray, subarr: np.ndarray) -> None:
         """
@@ -284,13 +271,6 @@ def _assert_safe_casting(cls, data: np.ndarray, subarr: np.ndarray) -> None:
             if not np.array_equal(data, subarr):
                 raise TypeError("Unsafe NumPy casting, you must explicitly cast")
 
-    @property
-    def _is_all_dates(self) -> bool:
-        """
-        Checks that all the labels are datetime objects.
-        """
-        return False
-
     def _format_native_types(
         self, *, na_rep="", float_format=None, decimal=".", quoting=None, **kwargs
     ):

pandas/core/internals/blocks.py

@@ -1376,6 +1376,8 @@ def where(self, other, cond) -> list[Block]:
 
         cond = extract_bool_array(cond)
 
+        orig_other = other
+        orig_cond = cond
         other = self._maybe_squeeze_arg(other)
         cond = self._maybe_squeeze_arg(cond)
 
@@ -1395,21 +1397,15 @@ def where(self, other, cond) -> list[Block]:
 
             if is_interval_dtype(self.dtype):
                 # TestSetitemFloatIntervalWithIntIntervalValues
-                blk = self.coerce_to_target_dtype(other)
-                if blk.dtype == _dtype_obj:
-                    # For now at least only support casting e.g.
-                    #  Interval[int64]->Interval[float64]
-                    raise
-                return blk.where(other, cond)
+                blk = self.coerce_to_target_dtype(orig_other)
+                nbs = blk.where(orig_other, orig_cond)
+                return self._maybe_downcast(nbs, "infer")
 
             elif isinstance(self, NDArrayBackedExtensionBlock):
                 # NB: not (yet) the same as
                 #  isinstance(values, NDArrayBackedExtensionArray)
-                if isinstance(self.dtype, PeriodDtype):
-                    # TODO: don't special-case
-                    raise
-                blk = self.coerce_to_target_dtype(other)
-                nbs = blk.where(other, cond)
+                blk = self.coerce_to_target_dtype(orig_other)
+                nbs = blk.where(orig_other, orig_cond)
                 return self._maybe_downcast(nbs, "infer")
 
             else:
@@ -1426,6 +1422,8 @@ def putmask(self, mask, new) -> list[Block]:
 
         values = self.values
 
+        orig_new = new
+        orig_mask = mask
         new = self._maybe_squeeze_arg(new)
         mask = self._maybe_squeeze_arg(mask)
 
@@ -1438,21 +1436,14 @@ def putmask(self, mask, new) -> list[Block]:
             if is_interval_dtype(self.dtype):
                 # Discussion about what we want to support in the general
                 #  case GH#39584
-                blk = self.coerce_to_target_dtype(new)
-                if blk.dtype == _dtype_obj:
-                    # For now at least, only support casting e.g.
-                    #  Interval[int64]->Interval[float64],
-                    raise
-                return blk.putmask(mask, new)
+                blk = self.coerce_to_target_dtype(orig_new)
+                return blk.putmask(orig_mask, orig_new)
 
             elif isinstance(self, NDArrayBackedExtensionBlock):
                 # NB: not (yet) the same as
                 #  isinstance(values, NDArrayBackedExtensionArray)
-                if isinstance(self.dtype, PeriodDtype):
-                    # TODO: don't special-case
-                    raise
-                blk = self.coerce_to_target_dtype(new)
-                return blk.putmask(mask, new)
+                blk = self.coerce_to_target_dtype(orig_new)
+                return blk.putmask(orig_mask, orig_new)
 
             else:
                 raise

pandas/tests/arrays/interval/test_interval.py

@@ -76,10 +76,16 @@ def test_set_closed(self, closed, new_closed):
         ],
     )
     def test_where_raises(self, other):
+        # GH#45768 The IntervalArray methods raises; the Series method coerces
         ser = pd.Series(IntervalArray.from_breaks([1, 2, 3, 4], closed="left"))
+        mask = np.array([True, False, True])
         match = "'value.closed' is 'right', expected 'left'."
         with pytest.raises(ValueError, match=match):
-            ser.where([True, False, True], other=other)
+            ser.array._where(mask, other)
+
+        res = ser.where(mask, other=other)
+        expected = ser.astype(object).where(mask, other)
+        tm.assert_series_equal(res, expected)
 
     def test_shift(self):
         # https://github.com/pandas-dev/pandas/issues/31495, GH#22428, GH#31502

pandas/tests/arrays/test_period.py

@@ -124,10 +124,16 @@ def test_sub_period():
     [pd.Period("2000", freq="H"), period_array(["2000", "2001", "2000"], freq="H")],
 )
 def test_where_different_freq_raises(other):
+    # GH#45768 The PeriodArray method raises, the Series method coerces
     ser = pd.Series(period_array(["2000", "2001", "2002"], freq="D"))
     cond = np.array([True, False, True])
+
     with pytest.raises(IncompatibleFrequency, match="freq"):
-        ser.where(cond, other)
+        ser.array._where(cond, other)
+
+    res = ser.where(cond, other)
+    expected = ser.astype(object).where(cond, other)
+    tm.assert_series_equal(res, expected)
 
 
 # ----------------------------------------------------------------------------

pandas/tests/frame/indexing/test_where.py

@@ -706,6 +706,20 @@ def test_where_interval_noop(self):
         res = ser.where(ser.notna())
         tm.assert_series_equal(res, ser)
 
+    def test_where_interval_fullop_downcast(self, frame_or_series):
+        # GH#45768
+        obj = frame_or_series([pd.Interval(0, 0)] * 2)
+        other = frame_or_series([1.0, 2.0])
+        res = obj.where(~obj.notna(), other)
+
+        # since all entries are being changed, we will downcast result
+        #  from object to ints (not floats)
+        tm.assert_equal(res, other.astype(np.int64))
+
+        # unlike where, Block.putmask does not downcast
+        obj.mask(obj.notna(), other, inplace=True)
+        tm.assert_equal(obj, other.astype(object))
+
     @pytest.mark.parametrize(
         "dtype",
         [
@@ -736,6 +750,16 @@ def test_where_datetimelike_noop(self, dtype):
         res4 = df.mask(mask2, "foo")
         tm.assert_frame_equal(res4, df)
 
+        # opposite case where we are replacing *all* values -> we downcast
+        #  from object dtype # GH#45768
+        res5 = df.where(mask2, 4)
+        expected = DataFrame(4, index=df.index, columns=df.columns)
+        tm.assert_frame_equal(res5, expected)
+
+        # unlike where, Block.putmask does not downcast
+        df.mask(~mask2, 4, inplace=True)
+        tm.assert_frame_equal(df, expected.astype(object))
+
 
 def test_where_try_cast_deprecated(frame_or_series):
     obj = DataFrame(np.random.randn(4, 3))
@@ -894,14 +918,29 @@ def test_where_period_invalid_na(frame_or_series, as_cat, request):
     else:
         msg = "value should be a 'Period'"
 
-    with pytest.raises(TypeError, match=msg):
-        obj.where(mask, tdnat)
+    if as_cat:
+        with pytest.raises(TypeError, match=msg):
+            obj.where(mask, tdnat)
 
-    with pytest.raises(TypeError, match=msg):
-        obj.mask(mask, tdnat)
+        with pytest.raises(TypeError, match=msg):
+            obj.mask(mask, tdnat)
+
+        with pytest.raises(TypeError, match=msg):
+            obj.mask(mask, tdnat, inplace=True)
+
+    else:
+        # With PeriodDtype, ser[i] = tdnat coerces instead of raising,
+        #  so for consistency, ser[mask] = tdnat must as well
+        expected = obj.astype(object).where(mask, tdnat)
+        result = obj.where(mask, tdnat)
+        tm.assert_equal(result, expected)
+
+        expected = obj.astype(object).mask(mask, tdnat)
+        result = obj.mask(mask, tdnat)
+        tm.assert_equal(result, expected)
 
-    with pytest.raises(TypeError, match=msg):
         obj.mask(mask, tdnat, inplace=True)
+        tm.assert_equal(obj, expected)
 
 
 def test_where_nullable_invalid_na(frame_or_series, any_numeric_ea_dtype):

pandas/tests/io/parser/test_unsupported.py

@@ -12,7 +12,10 @@
 
 import pytest
 
-from pandas.compat import is_platform_windows
+from pandas.compat import (
+    is_platform_mac,
+    is_platform_windows,
+)
 from pandas.errors import ParserError
 
 import pandas._testing as tm
@@ -174,9 +177,9 @@ def test_close_file_handle_on_invalid_usecols(all_parsers):
     if parser.engine == "pyarrow":
         pyarrow = pytest.importorskip("pyarrow")
         error = pyarrow.lib.ArrowKeyError
-        if is_platform_windows():
-            # GH#45547 causes timeouts on windows builds
-            pytest.skip("GH#45547 causing timeouts on windows builds 2022-01-22")
+        if is_platform_windows() or is_platform_mac():
+            # GH#45547 causes timeouts on windows/mac builds
+            pytest.skip("GH#45547 causing timeouts on windows/mac builds 2022-01-22")
 
     with tm.ensure_clean("test.csv") as fname:
         Path(fname).write_text("col1,col2\na,b\n1,2")

pandas/tests/series/indexing/test_setitem.py

@@ -18,6 +18,7 @@
     IntervalIndex,
     MultiIndex,
     NaT,
+    Period,
     Series,
     Timedelta,
     Timestamp,
@@ -1317,6 +1318,22 @@ def obj(self):
         return Series(timedelta_range("1 day", periods=4))
 
 
+@pytest.mark.parametrize(
+    "val", ["foo", Period("2016", freq="Y"), Interval(1, 2, closed="both")]
+)
+@pytest.mark.parametrize("exp_dtype", [object])
+class TestPeriodIntervalCoercion(CoercionTest):
+    # GH#45768
+    @pytest.fixture(
+        params=[
+            period_range("2016-01-01", periods=3, freq="D"),
+            interval_range(1, 5),
+        ]
+    )
+    def obj(self, request):
+        return Series(request.param)
+
+
 def test_20643():
     # closed by GH#45121
     orig = Series([0, 1, 2], index=["a", "b", "c"])