PDEP6 implementation pt 1: block.setitem, block.putmask (#50626)

* pt1

* fixup test collection

* fixup warnings

* add comments

* fixup warnings

* fixup test_indexing

* fixup test_set_value

* fixup test_where

* fixup test_asof

* add one more explicit upcast

* fixup test_update

* fixup test_constructors

* fixup test_stack_unstack

* catch warnings in test_update_dtypes

* fixup all test_update

* start fixing up setitem

* finish fixing up test_setitem

* more fixups

* catch numpy-dev warning

* fixup some more

* fixup test_indexing

* fixup test_function

* fixup test_multi;

* fixup test_base

* fixup test_impl

* fixup multiindex/test_setitem

* fixup test_scalar

* fixup test_loc

* fixup test_iloc

* fixup test_at

* fixup test_groupby

* fixup some doc warnings

* post-merge fixup

* change dtype in doctest

* fixup doctest

* explicit cast in test

* fixup test for COW

* fixup COW

* catch warnings in testsetitemcastingequivalents

* wip

* fixup setitem test int key!

* getting there!

* fixup test_setitem

* getting there

* fixup remaining warnings

* fix test_update

* fixup some failing test

* one more

* simplify

* simplify and remove some false-positives

* clean up

* remove final filterwarnings

* undo unrelated change

* fixup raises_chained_assignment_error

* remove another filterwarnings

* fixup interchange test

* better parametrisation

* okwarning => codeblock

* okwarning => codeblock in v1.3.0

* one more codeblock

* avoid upcast

* post-merge fixup

* docs fixup;

* post-merge fixup

* remove more upcasts

* adapt test from EA types

* move test to series/indexing

* add tests about warnings

* fixup tests

* add dataframe tests too

* fixup tests

* simplify

* try-fix docs build

* post-merge fixup

* raise assertionerror if self.dtype equals new_dtype

* add todo for test case which should warn

* add more todos

* post-merge fixup

* move logic for determining warning into the test

* uncomment test

---------

Co-authored-by: MarcoGorelli <>

Author: MarcoGorelli

doc/source/user_guide/categorical.rst

@@ -779,6 +779,7 @@ Setting values by assigning categorical data will also check that the ``categori
 Assigning a ``Categorical`` to parts of a column of other types will use the values:
 
 .. ipython:: python
+    :okwarning:
 
     df = pd.DataFrame({"a": [1, 1, 1, 1, 1], "b": ["a", "a", "a", "a", "a"]})
     df.loc[1:2, "a"] = pd.Categorical(["b", "b"], categories=["a", "b"])

doc/source/whatsnew/v0.21.0.rst

@@ -425,13 +425,12 @@ Note that this also changes the sum of an empty ``Series``. Previously this alwa
    In [1]: pd.Series([]).sum()
    Out[1]: 0
 
-but for consistency with the all-NaN case, this was changed to return NaN as well:
+but for consistency with the all-NaN case, this was changed to return 0 as well:
 
-.. ipython:: python
-   :okwarning:
-
-   pd.Series([]).sum()
+.. code-block:: ipython
 
+   In [2]: pd.Series([]).sum()
+   Out[2]: 0
 
 .. _whatsnew_0210.api_breaking.loc:
 
@@ -755,10 +754,16 @@ Previously assignments, ``.where()`` and ``.fillna()`` with a ``bool`` assignmen
 
 New behavior
 
-.. ipython:: python
+.. code-block:: ipython
 
-   s[1] = True
-   s
+   In [7]: s[1] = True
+
+   In [8]: s
+   Out[8]:
+   0       1
+   1    True
+   2       3
+   Length: 3, dtype: object
 
 Previously, as assignment to a datetimelike with a non-datetimelike would coerce the
 non-datetime-like item being assigned (:issue:`14145`).

doc/source/whatsnew/v1.3.0.rst

@@ -501,13 +501,17 @@ Consistent casting with setting into Boolean Series
 Setting non-boolean values into a :class:`Series` with ``dtype=bool`` now consistently
 casts to ``dtype=object`` (:issue:`38709`)
 
-.. ipython:: python
+.. code-block:: ipython
+
+   In [1]: orig = pd.Series([True, False])
+
+   In [2]: ser = orig.copy()
+
+   In [3]: ser.iloc[1] = np.nan
 
-   orig = pd.Series([True, False])
-   ser = orig.copy()
-   ser.iloc[1] = np.nan
-   ser2 = orig.copy()
-   ser2.iloc[1] = 2.0
+   In [4]: ser2 = orig.copy()
+
+   In [5]: ser2.iloc[1] = 2.0
 
 *Previous behavior*:
 
@@ -527,10 +531,19 @@ casts to ``dtype=object`` (:issue:`38709`)
 
 *New behavior*:
 
-.. ipython:: python
+.. code-block:: ipython
+
+   In [1]: ser
+   Out [1]:
+   0    True
+   1     NaN
+   dtype: object
 
-   ser
-   ser2
+   In [2]:ser2
+   Out [2]:
+   0    True
+   1     2.0
+   dtype: object
 
 
 .. _whatsnew_130.notable_bug_fixes.rolling_groupby_column:

pandas/core/internals/blocks.py

@@ -443,7 +443,7 @@ def split_and_operate(self, func, *args, **kwargs) -> list[Block]:
     # Up/Down-casting
 
     @final
-    def coerce_to_target_dtype(self, other) -> Block:
+    def coerce_to_target_dtype(self, other, warn_on_upcast: bool = False) -> Block:
         """
         coerce the current block to a dtype compat for other
         we will return a block, possibly object, and not raise
@@ -452,7 +452,21 @@ def coerce_to_target_dtype(self, other) -> Block:
         and will receive the same block
         """
         new_dtype = find_result_type(self.values.dtype, other)
-
+        if warn_on_upcast:
+            warnings.warn(
+                f"Setting an item of incompatible dtype is deprecated "
+                "and will raise in a future error of pandas. "
+                f"Value '{other}' has dtype incompatible with {self.values.dtype}, "
+                "please explicitly cast to a compatible dtype first.",
+                FutureWarning,
+                stacklevel=find_stack_level(),
+            )
+        if self.dtype == new_dtype:
+            raise AssertionError(
+                f"Did not expect new dtype {new_dtype} to equal self.dtype "
+                f"{self.values.dtype}. Please report a bug at "
+                "https://github.com/pandas-dev/pandas/issues."
+            )
         return self.astype(new_dtype, copy=False)
 
     @final
@@ -1111,7 +1125,7 @@ def setitem(self, indexer, value, using_cow: bool = False) -> Block:
             casted = np_can_hold_element(values.dtype, value)
         except LossySetitemError:
             # current dtype cannot store value, coerce to common dtype
-            nb = self.coerce_to_target_dtype(value)
+            nb = self.coerce_to_target_dtype(value, warn_on_upcast=True)
             return nb.setitem(indexer, value)
         else:
             if self.dtype == _dtype_obj:
@@ -1177,7 +1191,9 @@ def putmask(self, mask, new, using_cow: bool = False) -> list[Block]:
 
                 if not is_list_like(new):
                     # using just new[indexer] can't save us the need to cast
-                    return self.coerce_to_target_dtype(new).putmask(mask, new)
+                    return self.coerce_to_target_dtype(
+                        new, warn_on_upcast=True
+                    ).putmask(mask, new)
                 else:
                     indexer = mask.nonzero()[0]
                     nb = self.setitem(indexer, new[indexer], using_cow=using_cow)

pandas/tests/copy_view/test_indexing.py

@@ -925,11 +925,20 @@ def test_column_as_series_set_with_upcast(
             s[0] = "foo"
         expected = Series([1, 2, 3], name="a")
     elif using_copy_on_write or using_array_manager:
-        s[0] = "foo"
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            s[0] = "foo"
         expected = Series(["foo", 2, 3], dtype=object, name="a")
     else:
         with pd.option_context("chained_assignment", "warn"):
-            with tm.assert_produces_warning(SettingWithCopyWarning):
+            msg = "|".join(
+                [
+                    "A value is trying to be set on a copy of a slice from a DataFrame",
+                    "Setting an item of incompatible dtype is deprecated",
+                ]
+            )
+            with tm.assert_produces_warning(
+                (SettingWithCopyWarning, FutureWarning), match=msg
+            ):
                 s[0] = "foo"
         expected = Series(["foo", 2, 3], dtype=object, name="a")
 
@@ -1020,7 +1029,10 @@ def test_dataframe_add_column_from_series(backend, using_copy_on_write):
     ],
 )
 def test_set_value_copy_only_necessary_column(
-    using_copy_on_write, indexer_func, indexer, val
+    using_copy_on_write,
+    indexer_func,
+    indexer,
+    val,
 ):
     # When setting inplace, only copy column that is modified instead of the whole
     # block (by splitting the block)
@@ -1029,7 +1041,13 @@ def test_set_value_copy_only_necessary_column(
     df_orig = df.copy()
     view = df[:]
 
-    indexer_func(df)[indexer] = val
+    if val == "a" and indexer[0] != slice(None):
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype is deprecated"
+        ):
+            indexer_func(df)[indexer] = val
+    else:
+        indexer_func(df)[indexer] = val
 
     if using_copy_on_write:
         assert np.shares_memory(get_array(df, "b"), get_array(view, "b"))

pandas/tests/copy_view/test_methods.py

@@ -1405,15 +1405,18 @@ def test_putmask_aligns_rhs_no_reference(using_copy_on_write, dtype):
         assert np.shares_memory(arr_a, get_array(df, "a"))
 
 
-@pytest.mark.parametrize("val, exp", [(5.5, True), (5, False)])
-def test_putmask_dont_copy_some_blocks(using_copy_on_write, val, exp):
+@pytest.mark.parametrize(
+    "val, exp, warn", [(5.5, True, FutureWarning), (5, False, None)]
+)
+def test_putmask_dont_copy_some_blocks(using_copy_on_write, val, exp, warn):
     df = DataFrame({"a": [1, 2], "b": 1, "c": 1.5})
     view = df[:]
     df_orig = df.copy()
     indexer = DataFrame(
         [[True, False, False], [True, False, False]], columns=list("abc")
     )
-    df[indexer] = val
+    with tm.assert_produces_warning(warn, match="incompatible dtype"):
+        df[indexer] = val
 
     if using_copy_on_write:
         assert not np.shares_memory(get_array(view, "a"), get_array(df, "a"))

pandas/tests/frame/indexing/test_coercion.py

@@ -51,19 +51,31 @@ def test_37477():
     expected = DataFrame({"A": [1, 2, 3], "B": [3, 1.2, 5]})
 
     df = orig.copy()
-    df.at[1, "B"] = 1.2
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.at[1, "B"] = 1.2
     tm.assert_frame_equal(df, expected)
 
     df = orig.copy()
-    df.loc[1, "B"] = 1.2
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.loc[1, "B"] = 1.2
     tm.assert_frame_equal(df, expected)
 
     df = orig.copy()
-    df.iat[1, 1] = 1.2
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.iat[1, 1] = 1.2
     tm.assert_frame_equal(df, expected)
 
     df = orig.copy()
-    df.iloc[1, 1] = 1.2
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.iloc[1, 1] = 1.2
     tm.assert_frame_equal(df, expected)
 
 
@@ -94,11 +106,17 @@ def test_26395(indexer_al):
     expected = DataFrame({"D": [0, 0, 2]}, index=["A", "B", "C"], dtype=np.int64)
     tm.assert_frame_equal(df, expected)
 
-    indexer_al(df)["C", "D"] = 44.5
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        indexer_al(df)["C", "D"] = 44.5
     expected = DataFrame({"D": [0, 0, 44.5]}, index=["A", "B", "C"], dtype=np.float64)
     tm.assert_frame_equal(df, expected)
 
-    indexer_al(df)["C", "D"] = "hello"
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        indexer_al(df)["C", "D"] = "hello"
     expected = DataFrame({"D": [0, 0, "hello"]}, index=["A", "B", "C"], dtype=object)
     tm.assert_frame_equal(df, expected)
 

pandas/tests/frame/indexing/test_indexing.py

@@ -335,12 +335,18 @@ def test_setitem(self, float_frame, using_copy_on_write):
     def test_setitem2(self):
         # dtype changing GH4204
         df = DataFrame([[0, 0]])
-        df.iloc[0] = np.nan
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype"
+        ):
+            df.iloc[0] = np.nan
         expected = DataFrame([[np.nan, np.nan]])
         tm.assert_frame_equal(df, expected)
 
         df = DataFrame([[0, 0]])
-        df.loc[0] = np.nan
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype"
+        ):
+            df.loc[0] = np.nan
         tm.assert_frame_equal(df, expected)
 
     def test_setitem_boolean(self, float_frame):
@@ -1332,12 +1338,22 @@ def test_loc_expand_empty_frame_keep_midx_names(self):
         )
         tm.assert_frame_equal(df, expected)
 
-    @pytest.mark.parametrize("val", ["x", 1])
-    @pytest.mark.parametrize("idxr", ["a", ["a"]])
-    def test_loc_setitem_rhs_frame(self, idxr, val):
+    @pytest.mark.parametrize(
+        "val, idxr, warn",
+        [
+            ("x", "a", None),  # TODO: this should warn as well
+            ("x", ["a"], None),  # TODO: this should warn as well
+            (1, "a", None),  # TODO: this should warn as well
+            (1, ["a"], FutureWarning),
+        ],
+    )
+    def test_loc_setitem_rhs_frame(self, idxr, val, warn):
         # GH#47578
         df = DataFrame({"a": [1, 2]})
-        with tm.assert_produces_warning(None):
+
+        with tm.assert_produces_warning(
+            warn, match="Setting an item of incompatible dtype"
+        ):
             df.loc[:, idxr] = DataFrame({"a": [val, 11]}, index=[1, 2])
         expected = DataFrame({"a": [np.nan, val]})
         tm.assert_frame_equal(df, expected)
@@ -1537,8 +1553,11 @@ def test_setitem(self, uint64_frame):
         # With NaN: because uint64 has no NaN element,
         # the column should be cast to object.
         df2 = df.copy()
-        df2.iloc[1, 1] = pd.NaT
-        df2.iloc[1, 2] = pd.NaT
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype"
+        ):
+            df2.iloc[1, 1] = pd.NaT
+            df2.iloc[1, 2] = pd.NaT
         result = df2["B"]
         tm.assert_series_equal(notna(result), Series([True, False, True], name="B"))
         tm.assert_series_equal(
@@ -1851,3 +1870,54 @@ def test_setitem_dict_and_set_disallowed_multiindex(self, key):
         )
         with pytest.raises(TypeError, match="as an indexer is not supported"):
             df.loc[key] = 1
+
+
+class TestSetitemValidation:
+    # This is adapted from pandas/tests/arrays/masked/test_indexing.py
+    # but checks for warnings instead of errors.
+    def _check_setitem_invalid(self, df, invalid, indexer):
+        msg = "Setting an item of incompatible dtype is deprecated"
+        msg = re.escape(msg)
+
+        orig_df = df.copy()
+
+        # iloc
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            df.iloc[indexer, 0] = invalid
+            df = orig_df.copy()
+
+        # loc
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            df.loc[indexer, "a"] = invalid
+            df = orig_df.copy()
+
+    _invalid_scalars = [
+        1 + 2j,
+        "True",
+        "1",
+        "1.0",
+        pd.NaT,
+        np.datetime64("NaT"),
+        np.timedelta64("NaT"),
+    ]
+    _indexers = [0, [0], slice(0, 1), [True, False, False]]
+
+    @pytest.mark.parametrize(
+        "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)]
+    )
+    @pytest.mark.parametrize("indexer", _indexers)
+    def test_setitem_validation_scalar_bool(self, invalid, indexer):
+        df = DataFrame({"a": [True, False, False]}, dtype="bool")
+        self._check_setitem_invalid(df, invalid, indexer)
+
+    @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)])
+    @pytest.mark.parametrize("indexer", _indexers)
+    def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype, indexer):
+        df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype)
+        self._check_setitem_invalid(df, invalid, indexer)
+
+    @pytest.mark.parametrize("invalid", _invalid_scalars + [True])
+    @pytest.mark.parametrize("indexer", _indexers)
+    def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype, indexer):
+        df = DataFrame({"a": [1, 2, None]}, dtype=float_numpy_dtype)
+        self._check_setitem_invalid(df, invalid, indexer)