BUG raise pdep6 warning for loc full setter

doc/source/whatsnew/v2.2.0.rst

@@ -492,6 +492,7 @@ Conversion
 ^^^^^^^^^^
 - Bug in :func:`astype` when called with ``str`` on unpickled array - the array might change in-place (:issue:`54654`)
 - Bug in :meth:`Series.convert_dtypes` not converting all NA column to ``null[pyarrow]`` (:issue:`55346`)
+- Bug in ``DataFrame.loc`` was not throwing "incompatible dtype warning" (see PDEP6) when assigning a ``Series`` with a different dtype using a full column setter (e.g. ``df.loc[:, 'a'] = incompatible_value``) (:issue:`55791`)
 -
 
 Strings

pandas/core/frame.py

@@ -4157,7 +4157,7 @@ def _get_value(self, index, col, takeable: bool = False) -> Scalar:
         loc = engine.get_loc(index)
         return series._values[loc]
 
-    def isetitem(self, loc, value) -> None:
+    def isetitem(self, loc, value, *, inplace: bool = False) -> None:
         """
         Set the given value in the column with position `loc`.
 
@@ -4193,11 +4193,11 @@ def isetitem(self, loc, value) -> None:
 
             for i, idx in enumerate(loc):
                 arraylike, refs = self._sanitize_column(value.iloc[:, i])
-                self._iset_item_mgr(idx, arraylike, inplace=False, refs=refs)
+                self._iset_item_mgr(idx, arraylike, inplace=inplace, refs=refs)
             return
 
         arraylike, refs = self._sanitize_column(value)
-        self._iset_item_mgr(loc, arraylike, inplace=False, refs=refs)
+        self._iset_item_mgr(loc, arraylike, inplace=inplace, refs=refs)
 
     def __setitem__(self, key, value) -> None:
         if not PYPY and using_copy_on_write():
@@ -4423,7 +4423,7 @@ def _iset_item(self, loc: int, value: Series, inplace: bool = True) -> None:
                 loc, value._values, inplace=inplace, refs=value._references
             )
         else:
-            self._iset_item_mgr(loc, value._values.copy(), inplace=True)
+            self._iset_item_mgr(loc, value._values.copy(), inplace=inplace)
 
         # check if we are modifying a copy
         # try to set first as we want an invalid

pandas/core/indexing.py

@@ -2111,7 +2111,7 @@ def _setitem_single_column(self, loc: int, value, plane_indexer) -> None:
                 # If we're setting an entire column and we can't do it inplace,
                 #  then we can use value's dtype (or inferred dtype)
                 #  instead of object
-                self.obj.isetitem(loc, value)
+                self.obj.isetitem(loc, value, inplace=True)
         else:
             # set value into the column (first attempting to operate inplace, then
             #  falling back to casting if necessary)

pandas/core/internals/managers.py

@@ -1184,6 +1184,22 @@ def value_getitem(placement):
                     blk.set_inplace(blk_locs, value_getitem(val_locs))
                     continue
             else:
+                if inplace and blk.dtype != np.void:
+                    # Exclude np.void, as that is a special case for expansion.
+                    # We want to warn for
+                    #     df = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})
+                    #     df.loc[:, 'a'] = .3
+                    # but not for
+                    #     df = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})
+                    #     df.loc[:, 'b'] = .3
+                    warnings.warn(
+                        f"Setting an item of incompatible dtype is deprecated "
+                        "and will raise in a future error of pandas. "
+                        f"Value '{value}' has dtype incompatible with {blk.dtype}, "
+                        "please explicitly cast to a compatible dtype first.",
+                        FutureWarning,
+                        stacklevel=find_stack_level(),
+                    )
                 unfit_mgr_locs.append(blk.mgr_locs.as_array[blk_locs])
                 unfit_val_locs.append(val_locs)
 
@@ -1326,6 +1342,22 @@ def _iset_single(
             iloc = self.blklocs[loc]
             blk.set_inplace(slice(iloc, iloc + 1), value, copy=copy)
             return
+        elif inplace and blk.dtype != np.void:
+            # Exclude np.void, as that is a special case for expansion.
+            # We want to warn for
+            #     df = pd.DataFrame({'a': [1, 2]})
+            #     df.loc[:, 'a'] = .3
+            # but not for
+            #     df = pd.DataFrame({'a': [1, 2]})
+            #     df.loc[:, 'b'] = .3
+            warnings.warn(
+                f"Setting an item of incompatible dtype is deprecated "
+                "and will raise in a future error of pandas. "
+                f"Value '{value}' has dtype incompatible with {blk.dtype}, "
+                "please explicitly cast to a compatible dtype first.",
+                FutureWarning,
+                stacklevel=find_stack_level(),
+            )
 
         nb = new_block_2d(value, placement=blk._mgr_locs, refs=refs)
         old_blocks = self.blocks

pandas/io/stata.py

@@ -1790,7 +1790,7 @@ def read(
         if convert_dates:
             for i, fmt in enumerate(self._fmtlist):
                 if any(fmt.startswith(date_fmt) for date_fmt in _date_formats):
-                    data.iloc[:, i] = _stata_elapsed_date_to_datetime_vec(
+                    data[data.columns[i]] = _stata_elapsed_date_to_datetime_vec(
                         data.iloc[:, i], fmt
                     )
 
@@ -1866,7 +1866,7 @@ def _do_convert_missing(self, data: DataFrame, convert_missing: bool) -> DataFra
             replacements[i] = replacement
         if replacements:
             for idx, value in replacements.items():
-                data.iloc[:, idx] = value
+                data[data.columns[idx]] = value
         return data
 
     def _insert_strls(self, data: DataFrame) -> DataFrame:
@@ -1876,7 +1876,7 @@ def _insert_strls(self, data: DataFrame) -> DataFrame:
             if typ != "Q":
                 continue
             # Wrap v_o in a string to allow uint64 values as keys on 32bit OS
-            data.iloc[:, i] = [self.GSO[str(k)] for k in data.iloc[:, i]]
+            data[data.columns[i]] = [self.GSO[str(k)] for k in data.iloc[:, i]]
         return data
 
     def _do_select_columns(self, data: DataFrame, columns: Sequence[str]) -> DataFrame:

pandas/tests/copy_view/test_indexing.py

@@ -1129,12 +1129,16 @@ def test_set_value_copy_only_necessary_column(
     df_orig = df.copy()
     view = df[:]
 
-    if val == "a" and indexer[0] != slice(None):
-        # TODO(CoW-warn) assert the FutureWarning for CoW is also raised
+    if val == "a" and not warn_copy_on_write:
         with tm.assert_produces_warning(
             FutureWarning, match="Setting an item of incompatible dtype is deprecated"
         ):
             indexer_func(df)[indexer] = val
+    if val == "a" and warn_copy_on_write:
+        with tm.assert_produces_warning(
+            FutureWarning, match="incompatible dtype|Setting a value on a view"
+        ):
+            indexer_func(df)[indexer] = val
     else:
         with tm.assert_cow_warning(warn_copy_on_write):
             indexer_func(df)[indexer] = val

pandas/tests/frame/indexing/test_indexing.py

@@ -337,12 +337,14 @@ def test_setitem(self, float_frame, using_copy_on_write, warn_copy_on_write):
     def test_setitem2(self):
         # dtype changing GH4204
         df = DataFrame([[0, 0]])
-        df.iloc[0] = np.nan
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            df.iloc[0] = np.nan
         expected = DataFrame([[np.nan, np.nan]])
         tm.assert_frame_equal(df, expected)
 
         df = DataFrame([[0, 0]])
-        df.loc[0] = np.nan
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            df.loc[0] = np.nan
         tm.assert_frame_equal(df, expected)
 
     def test_setitem_boolean(self, float_frame):
@@ -934,7 +936,8 @@ def test_setitem_frame_upcast(self):
         # needs upcasting
         df = DataFrame([[1, 2, "foo"], [3, 4, "bar"]], columns=["A", "B", "C"])
         df2 = df.copy()
-        df2.loc[:, ["A", "B"]] = df.loc[:, ["A", "B"]] + 0.5
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            df2.loc[:, ["A", "B"]] = df.loc[:, ["A", "B"]] + 0.5
         expected = df.reindex(columns=["A", "B"])
         expected += 0.5
         expected["C"] = df["C"]
@@ -1372,33 +1375,35 @@ def test_loc_expand_empty_frame_keep_midx_names(self):
         tm.assert_frame_equal(df, expected)
 
     @pytest.mark.parametrize(
-        "val, idxr, warn",
+        "val, idxr",
         [
-            ("x", "a", None),  # TODO: this should warn as well
-            ("x", ["a"], None),  # TODO: this should warn as well
-            (1, "a", None),  # TODO: this should warn as well
-            (1, ["a"], FutureWarning),
+            ("x", "a"),
+            ("x", ["a"]),
+            (1, "a"),
+            (1, ["a"]),
         ],
     )
-    def test_loc_setitem_rhs_frame(self, idxr, val, warn):
+    def test_loc_setitem_rhs_frame(self, idxr, val):
         # GH#47578
         df = DataFrame({"a": [1, 2]})
 
         with tm.assert_produces_warning(
-            warn, match="Setting an item of incompatible dtype"
+            FutureWarning, match="Setting an item of incompatible dtype"
         ):
             df.loc[:, idxr] = DataFrame({"a": [val, 11]}, index=[1, 2])
         expected = DataFrame({"a": [np.nan, val]})
         tm.assert_frame_equal(df, expected)
 
     @td.skip_array_manager_invalid_test
-    def test_iloc_setitem_enlarge_no_warning(self, warn_copy_on_write):
+    def test_iloc_setitem_enlarge_no_warning(
+        self, warn_copy_on_write, using_copy_on_write
+    ):
         # GH#47381
         df = DataFrame(columns=["a", "b"])
         expected = df.copy()
         view = df[:]
-        # TODO(CoW-warn) false positive: shouldn't warn in case of enlargement?
-        with tm.assert_produces_warning(FutureWarning if warn_copy_on_write else None):
+        # TODO(pdep6-warn) false positive: shouldn't warn in case of enlargement?
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
             df.iloc[:, 0] = np.array([1, 2], dtype=np.float64)
         tm.assert_frame_equal(view, expected)
 
@@ -1963,7 +1968,7 @@ def _check_setitem_invalid(self, df, invalid, indexer, warn):
         np.datetime64("NaT"),
         np.timedelta64("NaT"),
     ]
-    _indexers = [0, [0], slice(0, 1), [True, False, False]]
+    _indexers = [0, [0], slice(0, 1), [True, False, False], slice(None, None, None)]
 
     @pytest.mark.parametrize(
         "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)]
@@ -1977,7 +1982,7 @@ def test_setitem_validation_scalar_bool(self, invalid, indexer):
     @pytest.mark.parametrize("indexer", _indexers)
     def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype, indexer):
         df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype)
-        if isna(invalid) and invalid is not pd.NaT:
+        if isna(invalid) and invalid is not pd.NaT and not np.isnat(invalid):
             warn = None
         else:
             warn = FutureWarning

pandas/tests/frame/indexing/test_setitem.py

@@ -860,7 +860,8 @@ def test_setitem_string_column_numpy_dtype_raising(self):
     def test_setitem_empty_df_duplicate_columns(self, using_copy_on_write):
         # GH#38521
         df = DataFrame(columns=["a", "b", "b"], dtype="float64")
-        df.loc[:, "a"] = list(range(2))
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            df.loc[:, "a"] = list(range(2))
         expected = DataFrame(
             [[0, np.nan, np.nan], [1, np.nan, np.nan]], columns=["a", "b", "b"]
         )
@@ -1367,3 +1368,23 @@ def test_frame_setitem_empty_dataframe(self):
             index=dti[:0],
         )
         tm.assert_frame_equal(df, expected)
+
+
+def test_full_setter_loc_incompatible_dtype():
+    # https://github.com/pandas-dev/pandas/issues/55791
+    df = DataFrame({"a": [1, 2]})
+    with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+        df.loc[:, "a"] = True
+    expected = DataFrame({"a": [True, True]})
+    tm.assert_frame_equal(df, expected)
+
+    df = DataFrame({"a": [1, 2]})
+    with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+        df.loc[:, "a"] = {0: 3.5, 1: 4.5}
+    expected = DataFrame({"a": [3.5, 4.5]})
+    tm.assert_frame_equal(df, expected)
+
+    df = DataFrame({"a": [1, 2]})
+    df.loc[:, "a"] = {0: 3, 1: 4}
+    expected = DataFrame({"a": [3, 4]})
+    tm.assert_frame_equal(df, expected)

pandas/tests/frame/methods/test_replace.py

@@ -1288,7 +1288,8 @@ def test_categorical_replace_with_dict(self, replace_dict, final_data):
         with pytest.raises(AssertionError, match=msg):
             # ensure non-inplace call does not affect original
             tm.assert_frame_equal(df, expected)
-        return_value = df.replace(replace_dict, 3, inplace=True)
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            return_value = df.replace(replace_dict, 3, inplace=True)
         assert return_value is None
         tm.assert_frame_equal(df, expected)
 

pandas/tests/frame/methods/test_update.py

@@ -144,11 +144,8 @@ def test_update_with_different_dtype(self, using_copy_on_write):
         # GH#3217
         df = DataFrame({"a": [1, 3], "b": [np.nan, 2]})
         df["c"] = np.nan
-        if using_copy_on_write:
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
             df.update({"c": Series(["foo"], index=[0])})
-        else:
-            with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
-                df["c"].update(Series(["foo"], index=[0]))
 
         expected = DataFrame({"a": [1, 3], "b": [np.nan, 2], "c": ["foo", np.nan]})
         tm.assert_frame_equal(df, expected)

pandas/tests/frame/test_constructors.py

@@ -2830,7 +2830,7 @@ def test_dict_data_arrow_column_expansion(self, key_val, col_vals, col_type):
         )
         result = DataFrame({key_val: [1, 2]}, columns=cols)
         expected = DataFrame([[1, np.nan], [2, np.nan]], columns=cols)
-        expected.iloc[:, 1] = expected.iloc[:, 1].astype(object)
+        expected[expected.columns[1]] = expected.iloc[:, 1].astype(object)
         tm.assert_frame_equal(result, expected)
 
 

pandas/tests/groupby/test_groupby.py

@@ -199,7 +199,7 @@ def f_1(grp):
     with tm.assert_produces_warning(FutureWarning, match=msg):
         result = df.groupby("A").apply(f_1)[["B"]]
     e = expected.copy()
-    e.loc["Tiger"] = np.nan
+    e["B"] = [3, 5, float("nan")]
     tm.assert_frame_equal(result, e)
 
     def f_2(grp):
@@ -211,7 +211,7 @@ def f_2(grp):
     with tm.assert_produces_warning(FutureWarning, match=msg):
         result = df.groupby("A").apply(f_2)[["B"]]
     e = expected.copy()
-    e.loc["Pony"] = np.nan
+    e["B"] = [3, float("nan"), 0]
     tm.assert_frame_equal(result, e)
 
     # 5592 revisited, with datetimes

pandas/tests/indexing/test_iloc.py

@@ -531,7 +531,8 @@ def test_iloc_setitem_frame_duplicate_columns_multiple_blocks(
 
         # if the assigned values cannot be held by existing integer arrays,
         #  we cast
-        df.iloc[:, 0] = df.iloc[:, 0] + 0.5
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            df.iloc[:, 0] = df.iloc[:, 0] + 0.5
         if not using_array_manager:
             assert len(df._mgr.blocks) == 2
 
@@ -1460,6 +1461,7 @@ def test_iloc_setitem_pure_position_based(self):
     def test_iloc_nullable_int64_size_1_nan(self):
         # GH 31861
         result = DataFrame({"a": ["test"], "b": [np.nan]})
-        result.loc[:, "b"] = result.loc[:, "b"].astype("Int64")
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            result.loc[:, "b"] = result.loc[:, "b"].astype("Int64")
         expected = DataFrame({"a": ["test"], "b": array([NA], dtype="Int64")})
         tm.assert_frame_equal(result, expected)

pandas/tests/indexing/test_indexing.py

@@ -828,7 +828,8 @@ def test_coercion_with_loc(self, expected):
         start_data, expected_result, warn = expected
 
         start_dataframe = DataFrame({"foo": start_data})
-        start_dataframe.loc[0, ["foo"]] = None
+        with tm.assert_produces_warning(warn, match="incompatible dtype"):
+            start_dataframe.loc[0, ["foo"]] = None
 
         expected_dataframe = DataFrame({"foo": expected_result})
         tm.assert_frame_equal(start_dataframe, expected_dataframe)
@@ -838,7 +839,8 @@ def test_coercion_with_setitem_and_dataframe(self, expected):
         start_data, expected_result, warn = expected
 
         start_dataframe = DataFrame({"foo": start_data})
-        start_dataframe[start_dataframe["foo"] == start_dataframe["foo"][0]] = None
+        with tm.assert_produces_warning(warn, match="incompatible dtype"):
+            start_dataframe[start_dataframe["foo"] == start_dataframe["foo"][0]] = None
 
         expected_dataframe = DataFrame({"foo": expected_result})
         tm.assert_frame_equal(start_dataframe, expected_dataframe)
@@ -848,7 +850,10 @@ def test_none_coercion_loc_and_dataframe(self, expected):
         start_data, expected_result, warn = expected
 
         start_dataframe = DataFrame({"foo": start_data})
-        start_dataframe.loc[start_dataframe["foo"] == start_dataframe["foo"][0]] = None
+        with tm.assert_produces_warning(warn, match="incompatible dtype"):
+            start_dataframe.loc[
+                start_dataframe["foo"] == start_dataframe["foo"][0]
+            ] = None
 
         expected_dataframe = DataFrame({"foo": expected_result})
         tm.assert_frame_equal(start_dataframe, expected_dataframe)
@@ -862,7 +867,8 @@ def test_none_coercion_mixed_dtypes(self):
                 "d": ["a", "b", "c"],
             }
         )
-        start_dataframe.iloc[0] = None
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            start_dataframe.iloc[0] = None
 
         exp = DataFrame(
             {