From ccfcb8d5f827c8155d11c2d582f1268c26098d3a Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 28 Dec 2022 14:12:05 +0000 Subject: [PATCH 01/80] pt1 --- pandas/core/internals/blocks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index aa859fac04921..6e0481d1551e8 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -11,6 +11,7 @@ cast, final, ) +import warnings import numpy as np @@ -35,6 +36,7 @@ ) from pandas.errors import AbstractMethodError from pandas.util._decorators import cache_readonly +from pandas.util._exceptions import find_stack_level from pandas.util._validators import validate_bool_kwarg from pandas.core.dtypes.astype import astype_array_safe @@ -941,6 +943,13 @@ def setitem(self, indexer, value) -> Block: casted = np_can_hold_element(values.dtype, value) except LossySetitemError: # current dtype cannot store value, coerce to common dtype + warnings.warn( + f"Setting an item of incompatible dtype is deprecated " + "and will raise in a future error of pandas. " + f"Value {value} has dtype incompatible with {values.dtype}", + FutureWarning, + stacklevel=find_stack_level(), + ) nb = self.coerce_to_target_dtype(value) return nb.setitem(indexer, value) else: From 30dde084a907dace377e1b9cd72ed3296ae11d8a Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 29 Dec 2022 13:49:02 +0000 Subject: [PATCH 02/80] fixup test collection --- pandas/tests/groupby/test_value_counts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/groupby/test_value_counts.py b/pandas/tests/groupby/test_value_counts.py index f67fea9cd6c0e..d744fcee5f8d1 100644 --- a/pandas/tests/groupby/test_value_counts.py +++ b/pandas/tests/groupby/test_value_counts.py @@ -55,6 +55,7 @@ def seed_df(seed_nans, n, m): "3rd": np.random.randint(1, m + 1, n), } ) + frame["3rd"] = frame["3rd"].astype("float64") if seed_nans: # Explicitly cast to float to avoid implicit cast when setting nan From d2dfa3cff5bbdf22d2cfc82a040442afa1348ac2 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 29 Dec 2022 17:13:25 +0000 Subject: [PATCH 03/80] fixup warnings --- pandas/tests/copy_view/test_indexing.py | 10 +++++++++- pandas/tests/frame/test_arithmetic.py | 2 +- pandas/tests/frame/test_constructors.py | 23 ++++++++++++++++++++--- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/pandas/tests/copy_view/test_indexing.py b/pandas/tests/copy_view/test_indexing.py index fd312c3375240..4a5fcf14ce01b 100644 --- a/pandas/tests/copy_view/test_indexing.py +++ b/pandas/tests/copy_view/test_indexing.py @@ -806,7 +806,15 @@ def test_column_as_series_set_with_upcast(using_copy_on_write, using_array_manag s[0] = "foo" else: with pd.option_context("chained_assignment", "warn"): - with tm.assert_produces_warning(SettingWithCopyWarning): + msg = "|".join( + [ + "A value is trying to be set on a copy of a slice from a DataFrame", + "Setting an item of incompatible dtype is deprecated", + ] + ) + with tm.assert_produces_warning( + (SettingWithCopyWarning, FutureWarning), match=msg + ): s[0] = "foo" expected = Series(["foo", 2, 3], dtype=object, name="a") diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index e7c5142bc05ca..e03af87dc7ab1 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -1935,7 +1935,7 @@ def test_dataframe_series_extension_dtypes(): def test_dataframe_blockwise_slicelike(): # GH#34367 arr = np.random.randint(0, 1000, (100, 10)) - df1 = DataFrame(arr) + df1 = DataFrame(arr, dtype="float") df2 = df1.copy() df2.iloc[0, [1, 3, 7]] = np.nan diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index e009ba45514a2..6e23740ce201d 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -26,7 +26,13 @@ from pandas.errors import IntCastingNaNError import pandas.util._test_decorators as td -from pandas.core.dtypes.common import is_integer_dtype +from pandas.core.dtypes.common import ( + is_complex_dtype, + is_datetime64_dtype, + is_float_dtype, + is_integer_dtype, + is_object_dtype, +) from pandas.core.dtypes.dtypes import ( DatetimeTZDtype, IntervalDtype, @@ -2529,8 +2535,19 @@ def check_views(c_only: bool = False): check_views() # TODO: most of the rest of this test belongs in indexing tests - df.iloc[0, 0] = 0 - df.iloc[0, 1] = 0 + if ( + is_float_dtype(df.dtypes[0]) + or is_integer_dtype(df.dtypes[0]) + or is_complex_dtype(df.dtypes[0]) + or is_object_dtype(df.dtypes[0]) + or is_datetime64_dtype(df.dtypes[0]) # TODO this one should warn + ): + warn = None + else: + warn = FutureWarning + with tm.assert_produces_warning(warn): + df.iloc[0, 0] = 0 + df.iloc[0, 1] = 0 if not copy: check_views(True) From e4ea519916751f234d456aa879a8d398af734c65 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 08:57:45 +0000 Subject: [PATCH 04/80] add comments From e160d7fd0dafd5e3d5c64db776cf6856573b1458 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 10:52:42 +0000 Subject: [PATCH 05/80] fixup warnings --- pandas/tests/frame/indexing/test_coercion.py | 30 ++++++++++++++++---- pandas/tests/frame/indexing/test_indexing.py | 18 +++++++++--- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/pandas/tests/frame/indexing/test_coercion.py b/pandas/tests/frame/indexing/test_coercion.py index 0e154d0e1d42b..ba0d8613b6228 100644 --- a/pandas/tests/frame/indexing/test_coercion.py +++ b/pandas/tests/frame/indexing/test_coercion.py @@ -51,19 +51,31 @@ def test_37477(): expected = DataFrame({"A": [1, 2, 3], "B": [3, 1.2, 5]}) df = orig.copy() - df.at[1, "B"] = 1.2 + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + df.at[1, "B"] = 1.2 tm.assert_frame_equal(df, expected) df = orig.copy() - df.loc[1, "B"] = 1.2 + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + df.loc[1, "B"] = 1.2 tm.assert_frame_equal(df, expected) df = orig.copy() - df.iat[1, 1] = 1.2 + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + df.iat[1, 1] = 1.2 tm.assert_frame_equal(df, expected) df = orig.copy() - df.iloc[1, 1] = 1.2 + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + df.iloc[1, 1] = 1.2 tm.assert_frame_equal(df, expected) @@ -94,11 +106,17 @@ def test_26395(indexer_al): expected = DataFrame({"D": [0, 0, 2]}, index=["A", "B", "C"], dtype=np.int64) tm.assert_frame_equal(df, expected) - indexer_al(df)["C", "D"] = 44.5 + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + indexer_al(df)["C", "D"] = 44.5 expected = DataFrame({"D": [0, 0, 44.5]}, index=["A", "B", "C"], dtype=np.float64) tm.assert_frame_equal(df, expected) - indexer_al(df)["C", "D"] = "hello" + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + indexer_al(df)["C", "D"] = "hello" expected = DataFrame({"D": [0, 0, "hello"]}, index=["A", "B", "C"], dtype=object) tm.assert_frame_equal(df, expected) diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index 57a132c1088c7..f19d33c30aac3 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -1317,12 +1317,22 @@ def test_loc_expand_empty_frame_keep_midx_names(self): ) tm.assert_frame_equal(df, expected) - @pytest.mark.parametrize("val", ["x", 1]) - @pytest.mark.parametrize("idxr", ["a", ["a"]]) - def test_loc_setitem_rhs_frame(self, idxr, val): + @pytest.mark.parametrize( + "val, idxr, warn", + [ + ("x", "a", None), + ("x", ["a"], None), + (1, "a", None), + (1, ["a"], FutureWarning), + ], + ) + def test_loc_setitem_rhs_frame(self, idxr, val, warn): # GH#47578 df = DataFrame({"a": [1, 2]}) - with tm.assert_produces_warning(None): + + with tm.assert_produces_warning( + warn, match="Setting an item of incompatible dtype" + ): df.loc[:, idxr] = DataFrame({"a": [val, 11]}, index=[1, 2]) expected = DataFrame({"a": [np.nan, val]}) tm.assert_frame_equal(df, expected) From f7971f07fb8c41a7e2813b1bac10c4e6c4cbf66f Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 12:51:34 +0000 Subject: [PATCH 06/80] fixup test_indexing --- pandas/tests/frame/indexing/test_indexing.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index f19d33c30aac3..bc4d1a38cbcb8 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -322,12 +322,18 @@ def test_setitem(self, float_frame, using_copy_on_write): def test_setitem2(self): # dtype changing GH4204 df = DataFrame([[0, 0]]) - df.iloc[0] = np.nan + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + df.iloc[0] = np.nan expected = DataFrame([[np.nan, np.nan]]) tm.assert_frame_equal(df, expected) df = DataFrame([[0, 0]]) - df.loc[0] = np.nan + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + df.loc[0] = np.nan tm.assert_frame_equal(df, expected) def test_setitem_boolean(self, float_frame): @@ -1533,8 +1539,11 @@ def test_setitem(self, uint64_frame): # With NaN: because uint64 has no NaN element, # the column should be cast to object. df2 = df.copy() - df2.iloc[1, 1] = pd.NaT - df2.iloc[1, 2] = pd.NaT + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + df2.iloc[1, 1] = pd.NaT + df2.iloc[1, 2] = pd.NaT result = df2["B"] tm.assert_series_equal(notna(result), Series([True, False, True], name="B")) tm.assert_series_equal( From 373a30ec430c62757211e54c51b1cc0bef0d15ae Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 12:53:50 +0000 Subject: [PATCH 07/80] fixup test_set_value --- pandas/tests/frame/indexing/test_set_value.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pandas/tests/frame/indexing/test_set_value.py b/pandas/tests/frame/indexing/test_set_value.py index 7b68566bab225..d7a17db8f27a3 100644 --- a/pandas/tests/frame/indexing/test_set_value.py +++ b/pandas/tests/frame/indexing/test_set_value.py @@ -6,6 +6,7 @@ DataFrame, isna, ) +import pandas._testing as tm class TestSetValue: @@ -26,11 +27,17 @@ def test_set_value_resize(self, float_frame): assert float_frame._get_value("foobar", "qux") == 0 res = float_frame.copy() - res._set_value("foobar", "baz", "sam") + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + res._set_value("foobar", "baz", "sam") assert res["baz"].dtype == np.object_ res = float_frame.copy() - res._set_value("foobar", "baz", True) + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + res._set_value("foobar", "baz", True) assert res["baz"].dtype == np.object_ res = float_frame.copy() @@ -38,7 +45,10 @@ def test_set_value_resize(self, float_frame): assert is_float_dtype(res["baz"]) assert isna(res["baz"].drop(["foobar"])).all() - res._set_value("foobar", "baz", "sam") + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype" + ): + res._set_value("foobar", "baz", "sam") assert res.loc["foobar", "baz"] == "sam" def test_set_value_with_index_dtype_change(self): From 26ab49d10a8b12c770d1765609c5a3290946f499 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 13:04:29 +0000 Subject: [PATCH 08/80] fixup test_where --- pandas/tests/frame/indexing/test_where.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pandas/tests/frame/indexing/test_where.py b/pandas/tests/frame/indexing/test_where.py index e37c881472b65..8d71b850cd678 100644 --- a/pandas/tests/frame/indexing/test_where.py +++ b/pandas/tests/frame/indexing/test_where.py @@ -493,7 +493,8 @@ def test_where_axis_with_upcast(self): tm.assert_frame_equal(result, expected) result = df.copy() - return_value = result.where(mask, ser, axis="index", inplace=True) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + return_value = result.where(mask, ser, axis="index", inplace=True) assert return_value is None tm.assert_frame_equal(result, expected) @@ -508,7 +509,8 @@ def test_where_axis_with_upcast(self): } ) result = df.copy() - return_value = result.where(mask, ser, axis="columns", inplace=True) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + return_value = result.where(mask, ser, axis="columns", inplace=True) assert return_value is None tm.assert_frame_equal(result, expected) @@ -559,11 +561,13 @@ def test_where_axis_multiple_dtypes(self): result = df.where(mask, d1, axis="index") tm.assert_frame_equal(result, expected) result = df.copy() - return_value = result.where(mask, d1, inplace=True) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + return_value = result.where(mask, d1, inplace=True) assert return_value is None tm.assert_frame_equal(result, expected) result = df.copy() - return_value = result.where(mask, d1, inplace=True, axis="index") + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + return_value = result.where(mask, d1, inplace=True, axis="index") assert return_value is None tm.assert_frame_equal(result, expected) From cc7cc5abbbd64509659fa854f17f29f485955cf8 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 13:19:14 +0000 Subject: [PATCH 09/80] fixup test_asof --- pandas/tests/frame/methods/test_asof.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/frame/methods/test_asof.py b/pandas/tests/frame/methods/test_asof.py index a08f8bf5f502e..a6374387b37e8 100644 --- a/pandas/tests/frame/methods/test_asof.py +++ b/pandas/tests/frame/methods/test_asof.py @@ -88,7 +88,8 @@ def test_missing(self, date_range_frame): ) tm.assert_series_equal(result, expected) - result = df.asof(to_datetime(["1989-12-31"])) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + result = df.asof(to_datetime(["1989-12-31"])) expected = DataFrame( index=to_datetime(["1989-12-31"]), columns=["A", "B"], dtype="float64" ) From 036248145e13d69b917bc04ec6c225abcdc0d2ab Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 13:46:45 +0000 Subject: [PATCH 10/80] add one more explicit upcast From 120aee78568a79340ee13b2ee615a542ebbfc282 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 13:50:35 +0000 Subject: [PATCH 11/80] fixup test_update --- pandas/tests/frame/methods/test_update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/frame/methods/test_update.py b/pandas/tests/frame/methods/test_update.py index ef468065f7e0e..0d9655f724321 100644 --- a/pandas/tests/frame/methods/test_update.py +++ b/pandas/tests/frame/methods/test_update.py @@ -148,7 +148,8 @@ def test_update_with_different_dtype(self, using_copy_on_write): if using_copy_on_write: df.update({"c": Series(["foo"], index=[0])}) else: - df["c"].update(Series(["foo"], index=[0])) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df["c"].update(Series(["foo"], index=[0])) expected = DataFrame({"a": [1, 3], "b": [np.nan, 2], "c": ["foo", np.nan]}) tm.assert_frame_equal(df, expected) From 7972c081f39f84cd8b5d2d6e5c3af002efa772f1 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 13:53:49 +0000 Subject: [PATCH 12/80] fixup test_constructors --- pandas/tests/frame/test_constructors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index 6e23740ce201d..58c305b12fea9 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -32,6 +32,7 @@ is_float_dtype, is_integer_dtype, is_object_dtype, + is_timedelta64_dtype, ) from pandas.core.dtypes.dtypes import ( DatetimeTZDtype, @@ -2541,6 +2542,7 @@ def check_views(c_only: bool = False): or is_complex_dtype(df.dtypes[0]) or is_object_dtype(df.dtypes[0]) or is_datetime64_dtype(df.dtypes[0]) # TODO this one should warn + or is_timedelta64_dtype(df.dtypes[0]) # TODO this one should warn ): warn = None else: From a16dcb84f2b3706c535e16dd2ae3643ec6810e3e Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 30 Dec 2022 14:25:59 +0000 Subject: [PATCH 13/80] fixup test_stack_unstack From 25dc26fbbe04fc8aa21a6871399b221eb0d96d30 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sat, 7 Jan 2023 14:51:45 +0000 Subject: [PATCH 14/80] catch warnings in test_update_dtypes --- pandas/tests/series/methods/test_update.py | 39 +++++++++++----------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/pandas/tests/series/methods/test_update.py b/pandas/tests/series/methods/test_update.py index 3e347604f7351..9ea85de9892e8 100644 --- a/pandas/tests/series/methods/test_update.py +++ b/pandas/tests/series/methods/test_update.py @@ -37,35 +37,36 @@ def test_update(self, using_copy_on_write): tm.assert_frame_equal(df, expected) @pytest.mark.parametrize( - "other, dtype, expected", + "other, dtype, expected, warn", [ # other is int - ([61, 63], "int32", Series([10, 61, 12], dtype="int32")), - ([61, 63], "int64", Series([10, 61, 12])), - ([61, 63], float, Series([10.0, 61.0, 12.0])), - ([61, 63], object, Series([10, 61, 12], dtype=object)), + ([61, 63], "int32", Series([10, 61, 12], dtype="int32"), None), + ([61, 63], "int64", Series([10, 61, 12]), None), + ([61, 63], float, Series([10.0, 61.0, 12.0]), None), + ([61, 63], object, Series([10, 61, 12], dtype=object), None), # other is float, but can be cast to int - ([61.0, 63.0], "int32", Series([10, 61, 12], dtype="int32")), - ([61.0, 63.0], "int64", Series([10, 61, 12])), - ([61.0, 63.0], float, Series([10.0, 61.0, 12.0])), - ([61.0, 63.0], object, Series([10, 61.0, 12], dtype=object)), + ([61.0, 63.0], "int32", Series([10, 61, 12], dtype="int32"), None), + ([61.0, 63.0], "int64", Series([10, 61, 12]), None), + ([61.0, 63.0], float, Series([10.0, 61.0, 12.0]), None), + ([61.0, 63.0], object, Series([10, 61.0, 12], dtype=object), None), # others is float, cannot be cast to int - ([61.1, 63.1], "int32", Series([10.0, 61.1, 12.0])), - ([61.1, 63.1], "int64", Series([10.0, 61.1, 12.0])), - ([61.1, 63.1], float, Series([10.0, 61.1, 12.0])), - ([61.1, 63.1], object, Series([10, 61.1, 12], dtype=object)), + ([61.1, 63.1], "int32", Series([10.0, 61.1, 12.0]), FutureWarning), + ([61.1, 63.1], "int64", Series([10.0, 61.1, 12.0]), FutureWarning), + ([61.1, 63.1], float, Series([10.0, 61.1, 12.0]), None), + ([61.1, 63.1], object, Series([10, 61.1, 12], dtype=object), None), # other is object, cannot be cast - ([(61,), (63,)], "int32", Series([10, (61,), 12])), - ([(61,), (63,)], "int64", Series([10, (61,), 12])), - ([(61,), (63,)], float, Series([10.0, (61,), 12.0])), - ([(61,), (63,)], object, Series([10, (61,), 12])), + ([(61,), (63,)], "int32", Series([10, (61,), 12]), FutureWarning), + ([(61,), (63,)], "int64", Series([10, (61,), 12]), FutureWarning), + ([(61,), (63,)], float, Series([10.0, (61,), 12.0]), FutureWarning), + ([(61,), (63,)], object, Series([10, (61,), 12]), None), ], ) - def test_update_dtypes(self, other, dtype, expected): + def test_update_dtypes(self, other, dtype, expected, warn): ser = Series([10, 11, 12], dtype=dtype) other = Series(other, index=[1, 3]) - ser.update(other) + with tm.assert_produces_warning(warn, match="item of incompatible dtype"): + ser.update(other) tm.assert_series_equal(ser, expected) From d45b312b9cf2fe8bdb842040dcbc967055276f60 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sat, 7 Jan 2023 14:52:34 +0000 Subject: [PATCH 15/80] fixup all test_update --- pandas/tests/series/methods/test_update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/series/methods/test_update.py b/pandas/tests/series/methods/test_update.py index 9ea85de9892e8..f15d8c0b940d9 100644 --- a/pandas/tests/series/methods/test_update.py +++ b/pandas/tests/series/methods/test_update.py @@ -27,7 +27,8 @@ def test_update(self, using_copy_on_write): df["c"] = np.nan df_orig = df.copy() - df["c"].update(Series(["foo"], index=[0])) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df["c"].update(Series(["foo"], index=[0])) if using_copy_on_write: expected = df_orig else: From e4ed8112774c7c208a2b917ec5bfe93a301ac900 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sat, 7 Jan 2023 15:07:15 +0000 Subject: [PATCH 16/80] start fixing up setitem --- pandas/tests/series/indexing/test_setitem.py | 115 +++++++++++++------ pandas/tests/series/indexing/test_where.py | 6 +- 2 files changed, 83 insertions(+), 38 deletions(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index fa196cf7d3f32..4cc7625168f09 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -536,22 +536,30 @@ def test_setitem_keep_precision(self, any_numeric_ea_dtype): expected = Series([1, 2, 10], dtype=any_numeric_ea_dtype) tm.assert_series_equal(ser, expected) - @pytest.mark.parametrize("indexer", [1, 2]) @pytest.mark.parametrize( - "na, target_na, dtype, target_dtype", + "na, target_na, dtype, target_dtype, indexer, warn", [ - (NA, NA, "Int64", "Int64"), - (NA, np.nan, "int64", "float64"), - (NaT, NaT, "int64", "object"), - (np.nan, NA, "Int64", "Int64"), - (np.nan, NA, "Float64", "Float64"), - (np.nan, np.nan, "int64", "float64"), + (NA, NA, "Int64", "Int64", 1, None), + (NA, NA, "Int64", "Int64", 2, None), + (NA, np.nan, "int64", "float64", 1, FutureWarning), + (NA, np.nan, "int64", "float64", 2, None), + (NaT, NaT, "int64", "object", 1, FutureWarning), + (NaT, NaT, "int64", "object", 2, None), + (np.nan, NA, "Int64", "Int64", 1, None), + (np.nan, NA, "Int64", "Int64", 2, None), + (np.nan, NA, "Float64", "Float64", 1, None), + (np.nan, NA, "Float64", "Float64", 2, None), + (np.nan, np.nan, "int64", "float64", 1, FutureWarning), + (np.nan, np.nan, "int64", "float64", 2, None), ], ) - def test_setitem_enlarge_with_na(self, na, target_na, dtype, target_dtype, indexer): + def test_setitem_enlarge_with_na( + self, na, target_na, dtype, target_dtype, indexer, warn + ): # GH#32346 ser = Series([1, 2], dtype=dtype) - ser[indexer] = na + with tm.assert_produces_warning(warn, match="item of incompatible dtype"): + ser[indexer] = na expected_values = [1, target_na] if indexer == 1 else [1, 2, target_na] expected = Series(expected_values, dtype=target_dtype) tm.assert_series_equal(ser, expected) @@ -640,7 +648,8 @@ def test_setitem_non_bool_into_bool(self, val, indexer_sli, unique): if not unique: ser.index = [1, 1] - indexer_sli(ser)[1] = val + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + indexer_sli(ser)[1] = val assert type(ser.iloc[1]) == type(val) expected = Series([True, val], dtype=object, index=ser.index) @@ -657,7 +666,8 @@ def test_setitem_boolean_array_into_npbool(self): ser[:2] = arr[:2] # no NAs -> can set inplace assert ser._values is values - ser[1:] = arr[1:] # has an NA -> cast to boolean dtype + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[1:] = arr[1:] # has an NA -> cast to boolean dtype expected = Series(arr) tm.assert_series_equal(ser, expected) @@ -707,6 +717,9 @@ def _check_inplace(self, is_inplace, orig, arr, obj): # otherwise original array should be unchanged tm.assert_equal(arr, orig._values) + @pytest.mark.filterwarnings( + "ignore:Setting an item of incompatible dtype:FutureWarning" + ) def test_int_key(self, obj, key, expected, val, indexer_sli, is_inplace): if not isinstance(key, int): return @@ -735,6 +748,9 @@ def test_int_key(self, obj, key, expected, val, indexer_sli, is_inplace): genkey = (x for x in [key]) self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) + @pytest.mark.filterwarnings( + "ignore:Setting an item of incompatible dtype:FutureWarning" + ) def test_slice_key(self, obj, key, expected, val, indexer_sli, is_inplace): if not isinstance(key, slice): return @@ -752,6 +768,9 @@ def test_slice_key(self, obj, key, expected, val, indexer_sli, is_inplace): genkey = (x for x in indkey) self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) + @pytest.mark.filterwarnings( + "ignore:Setting an item of incompatible dtype:FutureWarning" + ) def test_mask_key(self, obj, key, expected, val, indexer_sli): # setitem with boolean mask mask = np.zeros(obj.shape, dtype=bool) @@ -1373,42 +1392,51 @@ def test_20643(): expected = Series([0, 2.7, 2], index=["a", "b", "c"]) ser = orig.copy() - ser.at["b"] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.at["b"] = 2.7 tm.assert_series_equal(ser, expected) ser = orig.copy() - ser.loc["b"] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.loc["b"] = 2.7 tm.assert_series_equal(ser, expected) ser = orig.copy() - ser["b"] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser["b"] = 2.7 tm.assert_series_equal(ser, expected) ser = orig.copy() - ser.iat[1] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.iat[1] = 2.7 tm.assert_series_equal(ser, expected) ser = orig.copy() - ser.iloc[1] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.iloc[1] = 2.7 tm.assert_series_equal(ser, expected) orig_df = orig.to_frame("A") expected_df = expected.to_frame("A") df = orig_df.copy() - df.at["b", "A"] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df.at["b", "A"] = 2.7 tm.assert_frame_equal(df, expected_df) df = orig_df.copy() - df.loc["b", "A"] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df.loc["b", "A"] = 2.7 tm.assert_frame_equal(df, expected_df) df = orig_df.copy() - df.iloc[1, 0] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df.iloc[1, 0] = 2.7 tm.assert_frame_equal(df, expected_df) df = orig_df.copy() - df.iat[1, 0] = 2.7 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df.iat[1, 0] = 2.7 tm.assert_frame_equal(df, expected_df) @@ -1419,11 +1447,13 @@ def test_20643_comment(): expected = Series([np.nan, 1, 2], index=["a", "b", "c"]) ser = orig.copy() - ser.iat[0] = None + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.iat[0] = None tm.assert_series_equal(ser, expected) ser = orig.copy() - ser.iloc[0] = None + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.iloc[0] = None tm.assert_series_equal(ser, expected) @@ -1431,28 +1461,34 @@ def test_15413(): # fixed by GH#45121 ser = Series([1, 2, 3]) - ser[ser == 2] += 0.5 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[ser == 2] += 0.5 expected = Series([1, 2.5, 3]) tm.assert_series_equal(ser, expected) ser = Series([1, 2, 3]) - ser[1] += 0.5 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[1] += 0.5 tm.assert_series_equal(ser, expected) ser = Series([1, 2, 3]) - ser.loc[1] += 0.5 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.loc[1] += 0.5 tm.assert_series_equal(ser, expected) ser = Series([1, 2, 3]) - ser.iloc[1] += 0.5 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.iloc[1] += 0.5 tm.assert_series_equal(ser, expected) ser = Series([1, 2, 3]) - ser.iat[1] += 0.5 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.iat[1] += 0.5 tm.assert_series_equal(ser, expected) ser = Series([1, 2, 3]) - ser.at[1] += 0.5 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser.at[1] += 0.5 tm.assert_series_equal(ser, expected) @@ -1461,7 +1497,8 @@ def test_32878_int_itemsize(): arr = np.arange(5).astype("i4") ser = Series(arr) val = np.int64(np.iinfo(np.int64).max) - ser[0] = val + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[0] = val expected = Series([val, 1, 2, 3, 4], dtype=np.int64) tm.assert_series_equal(ser, expected) @@ -1473,7 +1510,8 @@ def test_32878_complex_itemsize(): val = val.astype("c16") # GH#32878 used to coerce val to inf+0.000000e+00j - ser[0] = val + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[0] = val assert ser[0] == val expected = Series([val, 1, 2, 3, 4], dtype="c16") tm.assert_series_equal(ser, expected) @@ -1482,7 +1520,8 @@ def test_32878_complex_itemsize(): def test_37692(indexer_al): # GH#37692 ser = Series([1, 2, 3], index=["a", "b", "c"]) - indexer_al(ser)["b"] = "test" + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + indexer_al(ser)["b"] = "test" expected = Series([1, "test", 3], index=["a", "b", "c"], dtype=object) tm.assert_series_equal(ser, expected) @@ -1494,11 +1533,13 @@ def test_setitem_bool_int_float_consistency(indexer_sli): # as the setitem can be done losslessly for dtype in [np.float64, np.int64]: ser = Series(0, index=range(3), dtype=dtype) - indexer_sli(ser)[0] = True + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + indexer_sli(ser)[0] = True assert ser.dtype == object ser = Series(0, index=range(3), dtype=bool) - ser[0] = dtype(1) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[0] = dtype(1) assert ser.dtype == object # 1.0 can be held losslessly, so no casting @@ -1516,7 +1557,8 @@ def test_setitem_positional_with_casting(): # we fallback we *also* get a ValueError if we try to set inplace. ser = Series([1, 2, 3], index=["a", "b", "c"]) - ser[0] = "X" + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[0] = "X" expected = Series(["X", 2, 3], index=["a", "b", "c"], dtype=object) tm.assert_series_equal(ser, expected) @@ -1525,7 +1567,8 @@ def test_setitem_positional_float_into_int_coerces(): # Case where we hit a KeyError and then trying to set in-place incorrectly # casts a float to an int ser = Series([1, 2, 3], index=["a", "b", "c"]) - ser[0] = 1.5 + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[0] = 1.5 expected = Series([1.5, 2, 3], index=["a", "b", "c"]) tm.assert_series_equal(ser, expected) diff --git a/pandas/tests/series/indexing/test_where.py b/pandas/tests/series/indexing/test_where.py index eabaf23bd36f8..785c707fc14e5 100644 --- a/pandas/tests/series/indexing/test_where.py +++ b/pandas/tests/series/indexing/test_where.py @@ -54,7 +54,8 @@ def test_where_unsafe_upcast(dtype, expected_dtype): values = [2.5, 3.5, 4.5, 5.5, 6.5] mask = s < 5 expected = Series(values + list(range(5, 10)), dtype=expected_dtype) - s[mask] = values + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + s[mask] = values tm.assert_series_equal(s, expected) @@ -66,7 +67,8 @@ def test_where_unsafe(): mask = s > 5 expected = Series(list(range(6)) + values, dtype="float64") - s[mask] = values + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + s[mask] = values tm.assert_series_equal(s, expected) # see gh-3235 From ace5e05425f1b5533d9e826f1224e33ed8499742 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sat, 7 Jan 2023 15:09:36 +0000 Subject: [PATCH 17/80] finish fixing up test_setitem --- pandas/tests/series/indexing/test_setitem.py | 39 +++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 4cc7625168f09..f88c83334ca5b 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -62,7 +62,10 @@ def test_setitem_multiindex_empty_slice(self): def test_setitem_with_string_index(self): # GH#23451 ser = Series([1, 2, 3], index=["Date", "b", "other"]) - ser["Date"] = date.today() + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + ser["Date"] = date.today() assert ser.Date == date.today() assert ser["Date"] == date.today() @@ -188,11 +191,14 @@ def test_setitem_series_object_dtype(self, indexer, ser_index): expected = Series([Series([42], index=[ser_index]), 0], dtype="object") tm.assert_series_equal(ser, expected) - @pytest.mark.parametrize("index, exp_value", [(0, 42), (1, np.nan)]) - def test_setitem_series(self, index, exp_value): + @pytest.mark.parametrize( + "index, exp_value, warn", [(0, 42, None), (1, np.nan, FutureWarning)] + ) + def test_setitem_series(self, index, exp_value, warn): # GH#38303 ser = Series([0, 0]) - ser.loc[0] = Series([42], index=[index]) + with tm.assert_produces_warning(warn, match="item of incompatible dtype"): + ser.loc[0] = Series([42], index=[index]) expected = Series([exp_value, 0]) tm.assert_series_equal(ser, expected) @@ -256,7 +262,10 @@ def test_setitem_mask_align_and_promote(self): mask = ts > 0 left = ts.copy() right = ts[mask].copy().map(str) - left[mask] = right + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + left[mask] = right expected = ts.map(lambda t: str(t) if t > 0 else t) tm.assert_series_equal(left, expected) @@ -264,7 +273,10 @@ def test_setitem_mask_promote_strs(self): ser = Series([0, 1, 2, 0]) mask = ser > 0 ser2 = ser[mask].map(str) - ser[mask] = ser2 + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + ser[mask] = ser2 expected = Series([0, "1", "2", 0]) tm.assert_series_equal(ser, expected) @@ -356,7 +368,10 @@ def test_setitem_with_bool_mask_and_values_matching_n_trues_in_length(self): def test_setitem_nan_with_bool(self): # GH 13034 result = Series([True, False, True]) - result[0] = np.nan + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + result[0] = np.nan expected = Series([np.nan, False, True], dtype=object) tm.assert_series_equal(result, expected) @@ -367,12 +382,18 @@ def test_setitem_mask_smallint_upcast(self): mask = np.array([True, False, True]) ser = orig.copy() - ser[mask] = Series(alt) + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + ser[mask] = Series(alt) expected = Series([999, 2, 1001]) tm.assert_series_equal(ser, expected) ser2 = orig.copy() - ser2.mask(mask, alt, inplace=True) + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + ser2.mask(mask, alt, inplace=True) tm.assert_series_equal(ser2, expected) ser3 = orig.copy() From 34a6194f07eaef6b79ec245b4ac3b55fae3d0daf Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sat, 7 Jan 2023 17:32:47 +0000 Subject: [PATCH 18/80] more fixups --- pandas/tests/series/indexing/test_where.py | 18 +++++++++--------- pandas/tests/series/test_missing.py | 3 ++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pandas/tests/series/indexing/test_where.py b/pandas/tests/series/indexing/test_where.py index 785c707fc14e5..4eb4378809b39 100644 --- a/pandas/tests/series/indexing/test_where.py +++ b/pandas/tests/series/indexing/test_where.py @@ -38,23 +38,23 @@ def test_where_unsafe_float(float_numpy_dtype): @pytest.mark.parametrize( - "dtype,expected_dtype", + "dtype,expected_dtype,warn", [ - (np.int8, np.float64), - (np.int16, np.float64), - (np.int32, np.float64), - (np.int64, np.float64), - (np.float32, np.float32), - (np.float64, np.float64), + (np.int8, np.float64, FutureWarning), + (np.int16, np.float64, FutureWarning), + (np.int32, np.float64, FutureWarning), + (np.int64, np.float64, FutureWarning), + (np.float32, np.float32, None), + (np.float64, np.float64, None), ], ) -def test_where_unsafe_upcast(dtype, expected_dtype): +def test_where_unsafe_upcast(dtype, expected_dtype, warn): # see gh-9743 s = Series(np.arange(10), dtype=dtype) values = [2.5, 3.5, 4.5, 5.5, 6.5] mask = s < 5 expected = Series(values + list(range(5, 10)), dtype=expected_dtype) - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + with tm.assert_produces_warning(warn, match="incompatible dtype"): s[mask] = values tm.assert_series_equal(s, expected) diff --git a/pandas/tests/series/test_missing.py b/pandas/tests/series/test_missing.py index 3c0f962b90086..b21f27fe75cd2 100644 --- a/pandas/tests/series/test_missing.py +++ b/pandas/tests/series/test_missing.py @@ -99,6 +99,7 @@ def test_hasnans_uncached_for_series(): ser = idx.to_series() assert ser.hasnans is False assert not hasattr(ser, "_cache") - ser.iloc[-1] = np.nan + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + ser.iloc[-1] = np.nan assert ser.hasnans is True assert Series.hasnans.__doc__ == Index.hasnans.__doc__ From 0cfefa4d64ac9096d8cad09a5d8b16be62ae118c Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sun, 8 Jan 2023 13:22:38 +0000 Subject: [PATCH 19/80] catch numpy-dev warning --- pandas/tests/series/indexing/test_setitem.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index f88c83334ca5b..cfcab8b458c26 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -1531,7 +1531,12 @@ def test_32878_complex_itemsize(): val = val.astype("c16") # GH#32878 used to coerce val to inf+0.000000e+00j - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + # 'overflow encountered in cast' thrown by numpy + with tm.assert_produces_warning( + (FutureWarning, RuntimeWarning), + match="incompatible dtype|overflow encountered in cast", + check_stacklevel=False, + ): ser[0] = val assert ser[0] == val expected = Series([val, 1, 2, 3, 4], dtype="c16") From 150fa9ae79bc522afa763c782148b9b2446dc617 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sun, 8 Jan 2023 13:28:46 +0000 Subject: [PATCH 20/80] fixup some more --- pandas/tests/groupby/test_filters.py | 15 ++++++++++----- pandas/tests/indexing/test_indexing.py | 5 ++++- pandas/tests/io/formats/test_format.py | 3 ++- pandas/tests/reshape/test_from_dummies.py | 6 ++++-- pandas/tests/window/test_timeseries_window.py | 3 ++- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/pandas/tests/groupby/test_filters.py b/pandas/tests/groupby/test_filters.py index 9db4a14929724..0097c5dd15cb1 100644 --- a/pandas/tests/groupby/test_filters.py +++ b/pandas/tests/groupby/test_filters.py @@ -346,7 +346,8 @@ def test_filter_and_transform_with_non_unique_int_index(): actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False) expected = df.copy() - expected.iloc[[0, 3, 5, 6]] = np.nan + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + expected.iloc[[0, 3, 5, 6]] = np.nan tm.assert_frame_equal(actual, expected) # Filter Series @@ -389,7 +390,8 @@ def test_filter_and_transform_with_multiple_non_unique_int_index(): actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False) expected = df.copy() - expected.iloc[[0, 3, 5, 6]] = np.nan + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + expected.iloc[[0, 3, 5, 6]] = np.nan tm.assert_frame_equal(actual, expected) # Filter Series @@ -432,7 +434,8 @@ def test_filter_and_transform_with_non_unique_float_index(): actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False) expected = df.copy() - expected.iloc[[0, 3, 5, 6]] = np.nan + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + expected.iloc[[0, 3, 5, 6]] = np.nan tm.assert_frame_equal(actual, expected) # Filter Series @@ -478,7 +481,8 @@ def test_filter_and_transform_with_non_unique_timestamp_index(): actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False) expected = df.copy() - expected.iloc[[0, 3, 5, 6]] = np.nan + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + expected.iloc[[0, 3, 5, 6]] = np.nan tm.assert_frame_equal(actual, expected) # Filter Series @@ -521,7 +525,8 @@ def test_filter_and_transform_with_non_unique_string_index(): actual = grouped_df.filter(lambda x: len(x) > 1, dropna=False) expected = df.copy() - expected.iloc[[0, 3, 5, 6]] = np.nan + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + expected.iloc[[0, 3, 5, 6]] = np.nan tm.assert_frame_equal(actual, expected) # Filter Series diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 910406b967770..458299b6d1271 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -846,7 +846,10 @@ def test_none_coercion_mixed_dtypes(self): "d": ["a", "b", "c"], } ) - start_dataframe.iloc[0] = None + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + start_dataframe.iloc[0] = None exp = DataFrame( { diff --git a/pandas/tests/io/formats/test_format.py b/pandas/tests/io/formats/test_format.py index d04e5183d980f..dee90afc37db7 100644 --- a/pandas/tests/io/formats/test_format.py +++ b/pandas/tests/io/formats/test_format.py @@ -193,7 +193,8 @@ def test_eng_float_formatter(self, float_frame): ) def test_show_counts(self, row, columns, show_counts, result): - df = DataFrame(1, columns=range(10), index=range(10)) + # Explicit cast to float to avoid implicit cast when setting nan + df = DataFrame(1, columns=range(10), index=range(10), dtype="float") df.iloc[1, 1] = np.nan with option_context( diff --git a/pandas/tests/reshape/test_from_dummies.py b/pandas/tests/reshape/test_from_dummies.py index ab80473725288..e2741a04dfeff 100644 --- a/pandas/tests/reshape/test_from_dummies.py +++ b/pandas/tests/reshape/test_from_dummies.py @@ -164,7 +164,8 @@ def test_error_with_prefix_default_category_dict_not_complete( def test_error_with_prefix_contains_nan(dummies_basic): - dummies_basic.loc[2, "col2_c"] = np.nan + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + dummies_basic.loc[2, "col2_c"] = np.nan with pytest.raises( ValueError, match=r"Dummy DataFrame contains NA value in column: 'col2_c'" ): @@ -172,7 +173,8 @@ def test_error_with_prefix_contains_nan(dummies_basic): def test_error_with_prefix_contains_non_dummies(dummies_basic): - dummies_basic.loc[2, "col2_c"] = "str" + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + dummies_basic.loc[2, "col2_c"] = "str" with pytest.raises(TypeError, match=r"Passed DataFrame contains non-dummy data"): from_dummies(dummies_basic, sep="_") diff --git a/pandas/tests/window/test_timeseries_window.py b/pandas/tests/window/test_timeseries_window.py index d04cdb3e46bc0..f33dfb2ca94ec 100644 --- a/pandas/tests/window/test_timeseries_window.py +++ b/pandas/tests/window/test_timeseries_window.py @@ -579,7 +579,8 @@ def test_ragged_max(self, ragged): def test_freqs_ops(self, freq, op, result_data): # GH 21096 index = date_range(start="2018-1-1 01:00:00", freq=f"1{freq}", periods=10) - s = Series(data=0, index=index) + # Explicit cast to float to avoid implicit cast when setting nan + s = Series(data=0, index=index, dtype="float") s.iloc[1] = np.nan s.iloc[-1] = 2 result = getattr(s.rolling(window=f"10{freq}"), op)() From 1cc3f509a9dd93ae6add775161126558aeb7cc85 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Sun, 8 Jan 2023 13:42:30 +0000 Subject: [PATCH 21/80] fixup test_indexing --- pandas/tests/indexing/test_indexing.py | 34 +++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 458299b6d1271..35251f5cff816 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -177,7 +177,10 @@ def test_setitem_dtype_upcast(self): df["c"] = np.nan assert df["c"].dtype == np.float64 - df.loc[0, "c"] = "foo" + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df.loc[0, "c"] = "foo" expected = DataFrame( [{"a": 1, "b": np.nan, "c": "foo"}, {"a": 3, "b": 2, "c": np.nan}] ) @@ -194,7 +197,10 @@ def test_setitem_dtype_upcast2(self, val): ) left = df.copy() - left.loc["a", "bar"] = val + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + left.loc["a", "bar"] = val right = DataFrame( [[0, val, 2], [3, 4, 5]], index=list("ab"), @@ -211,7 +217,10 @@ def test_setitem_dtype_upcast3(self): index=list("ab"), columns=["foo", "bar", "baz"], ) - left.loc["a", "bar"] = "wxyz" + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + left.loc["a", "bar"] = "wxyz" right = DataFrame( [[0, "wxyz", 0.2], [0.3, 0.4, 0.5]], @@ -455,7 +464,10 @@ def test_multi_assign(self): cols = ["col1", "col2"] dft = df2 * 2 - dft.iloc[3, 3] = np.nan + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + dft.iloc[3, 3] = np.nan expected = DataFrame( { @@ -467,7 +479,10 @@ def test_multi_assign(self): ) # frame on rhs - df2.loc[mask, cols] = dft.loc[mask, cols] + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df2.loc[mask, cols] = dft.loc[mask, cols] tm.assert_frame_equal(df2, expected) # with an ndarray on rhs @@ -482,7 +497,10 @@ def test_multi_assign(self): } ) df2 = df.copy() - df2.loc[mask, cols] = dft.loc[mask, cols].values + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df2.loc[mask, cols] = dft.loc[mask, cols].values tm.assert_frame_equal(df2, expected) def test_multi_assign_broadcasting_rhs(self): @@ -659,6 +677,7 @@ def test_loc_setitem_fullindex_views(self): df.loc[df.index] = df.loc[df.index] tm.assert_frame_equal(df, df2) + @pytest.mark.filterwarnings("ignore:.*item of incompatible dtype.*:FutureWarning") def test_rhs_alignment(self): # GH8258, tests that both rows & columns are aligned to what is # assigned to. covers both uniform data-type & multi-type cases @@ -808,6 +827,7 @@ class TestDataframeNoneCoercion: ] @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) + @pytest.mark.filterwarnings("ignore:.*item of incompatible dtype.*:FutureWarning") def test_coercion_with_loc(self, expected): start_data, expected_result = expected @@ -818,6 +838,7 @@ def test_coercion_with_loc(self, expected): tm.assert_frame_equal(start_dataframe, expected_dataframe) @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) + @pytest.mark.filterwarnings("ignore:.*item of incompatible dtype.*:FutureWarning") def test_coercion_with_setitem_and_dataframe(self, expected): start_data, expected_result = expected @@ -828,6 +849,7 @@ def test_coercion_with_setitem_and_dataframe(self, expected): tm.assert_frame_equal(start_dataframe, expected_dataframe) @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) + @pytest.mark.filterwarnings("ignore:.*item of incompatible dtype.*:FutureWarning") def test_none_coercion_loc_and_dataframe(self, expected): start_data, expected_result = expected From 3f15670f00c841b90e63f73ab39b4437e8290359 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 12:42:35 +0000 Subject: [PATCH 22/80] fixup test_function --- pandas/tests/groupby/test_function.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/tests/groupby/test_function.py b/pandas/tests/groupby/test_function.py index 875037b390883..da3bc2109341b 100644 --- a/pandas/tests/groupby/test_function.py +++ b/pandas/tests/groupby/test_function.py @@ -826,6 +826,8 @@ def test_cummin(dtypes_for_minmax): tm.assert_frame_equal(result, expected, check_exact=True) # Test nan in some values + # Explicit cast to float to avoid implicit cast when setting nan + base_df = base_df.astype({"B": "float"}) base_df.loc[[0, 2, 4, 6], "B"] = np.nan expected = DataFrame({"B": [np.nan, 4, np.nan, 2, np.nan, 3, np.nan, 1]}) result = base_df.groupby("A").cummin() @@ -891,6 +893,8 @@ def test_cummax(dtypes_for_minmax): tm.assert_frame_equal(result, expected) # Test nan in some values + # Explicit cast to float to avoid implicit cast when setting nan + base_df = base_df.astype({"B": "float"}) base_df.loc[[0, 2, 4, 6], "B"] = np.nan expected = DataFrame({"B": [np.nan, 4, np.nan, 4, np.nan, 3, np.nan, 3]}) result = base_df.groupby("A").cummax() From d0de3f135704ee1fcdafa5acdca7e3c369a84fea Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 12:45:39 +0000 Subject: [PATCH 23/80] fixup test_multi; --- pandas/tests/reshape/merge/test_multi.py | 3 ++- pandas/tests/window/test_rolling.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/tests/reshape/merge/test_multi.py b/pandas/tests/reshape/merge/test_multi.py index 0dbe45eeb1e82..293e5ea58e3b0 100644 --- a/pandas/tests/reshape/merge/test_multi.py +++ b/pandas/tests/reshape/merge/test_multi.py @@ -123,7 +123,8 @@ def run_asserts(left, right, sort): lc = list(map(chr, np.arange(ord("a"), ord("z") + 1))) left = DataFrame(np.random.choice(lc, (5000, 2)), columns=["1st", "3rd"]) - left.insert(1, "2nd", np.random.randint(0, 1000, len(left))) + # Explicit cast to float to avoid implicit cast when setting nan + left.insert(1, "2nd", np.random.randint(0, 1000, len(left)).astype("float")) i = np.random.permutation(len(left)) right = left.iloc[i].copy() diff --git a/pandas/tests/window/test_rolling.py b/pandas/tests/window/test_rolling.py index 60049d0ac633a..47b537cccfe84 100644 --- a/pandas/tests/window/test_rolling.py +++ b/pandas/tests/window/test_rolling.py @@ -432,7 +432,10 @@ def test_closed_uneven(): ) def test_closed_min_max_minp(func, closed, expected): # see gh-21704 - ser = Series(data=np.arange(10), index=date_range("2000", periods=10)) + # Explicit cast to float to avoid implicit cast when setting nan + ser = Series( + data=np.arange(10), index=date_range("2000", periods=10), dtype="float" + ) ser[ser.index[-3:]] = np.nan result = getattr(ser.rolling("3D", min_periods=2, closed=closed), func)() expected = Series(expected, index=ser.index) From dc6c60b090ec584c09e46787e089821f0b4ec2d0 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 12:49:49 +0000 Subject: [PATCH 24/80] fixup test_base --- pandas/tests/resample/test_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/resample/test_base.py b/pandas/tests/resample/test_base.py index 55e8c4e818ce3..19445c35f0bb6 100644 --- a/pandas/tests/resample/test_base.py +++ b/pandas/tests/resample/test_base.py @@ -68,7 +68,8 @@ def test_asfreq_fill_value(series, create_index): expected = ser.reindex(new_index) tm.assert_series_equal(result, expected) - frame = ser.to_frame("value") + # Explicit cast to float to avoid implicit cast when setting None + frame = ser.astype("float").to_frame("value") frame.iloc[1] = None result = frame.resample("1H").asfreq(fill_value=4.0) new_index = create_index(frame.index[0], frame.index[-1], freq="1H") From e33563a51e536286dc20ecde47c161aeeff2fdad Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 12:56:08 +0000 Subject: [PATCH 25/80] fixup test_impl --- pandas/tests/interchange/test_impl.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pandas/tests/interchange/test_impl.py b/pandas/tests/interchange/test_impl.py index 078a17510d502..a3add88c93da3 100644 --- a/pandas/tests/interchange/test_impl.py +++ b/pandas/tests/interchange/test_impl.py @@ -7,6 +7,8 @@ from pandas._libs.tslibs import iNaT import pandas.util._test_decorators as td +from pandas.core.dtypes.common import is_float_dtype + import pandas as pd import pandas._testing as tm from pandas.core.interchange.column import PandasColumn @@ -115,7 +117,12 @@ def test_missing_from_masked(): null_idx = df.index[ rng.choice(np.arange(len(df)), size=num_nulls, replace=False) ] - df.loc[null_idx, col] = None + if is_float_dtype(df[col]): + warn = None + else: + warn = FutureWarning + with tm.assert_produces_warning(warn, match="item of incompatible dtype"): + df.loc[null_idx, col] = None df2 = df.__dataframe__() From 37e05206a719cfb36825b4c0e88a07d0077c1b00 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 12:57:23 +0000 Subject: [PATCH 26/80] fixup multiindex/test_setitem --- pandas/tests/indexing/multiindex/test_setitem.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexing/multiindex/test_setitem.py b/pandas/tests/indexing/multiindex/test_setitem.py index ac10a6d82dc89..2f87d2e5b434e 100644 --- a/pandas/tests/indexing/multiindex/test_setitem.py +++ b/pandas/tests/indexing/multiindex/test_setitem.py @@ -220,7 +220,10 @@ def test_multiindex_assignment_single_dtype(self, using_copy_on_write): tm.assert_numpy_array_equal(view, exp.values) # arr + 0.5 cannot be cast losslessly to int, so we upcast - df.loc[4, "c"] = arr + 0.5 + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df.loc[4, "c"] = arr + 0.5 result = df.loc[4, "c"] exp = exp + 0.5 tm.assert_series_equal(result, exp) From 2dd85aba01d12b003db60a7e41d7815af9160ae2 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 12:58:11 +0000 Subject: [PATCH 27/80] fixup test_scalar --- pandas/tests/indexing/test_scalar.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexing/test_scalar.py b/pandas/tests/indexing/test_scalar.py index b1c3008b04d3e..a85611d498d7c 100644 --- a/pandas/tests/indexing/test_scalar.py +++ b/pandas/tests/indexing/test_scalar.py @@ -205,7 +205,10 @@ def test_mixed_index_at_iat_loc_iloc_dataframe(self): def test_iat_setter_incompatible_assignment(self): # GH 23236 result = DataFrame({"a": [0, 1], "b": [4, 5]}) - result.iat[0, 0] = None + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + result.iat[0, 0] = None expected = DataFrame({"a": [None, 1], "b": [4, 5]}) tm.assert_frame_equal(result, expected) From 24ca7c2d000ca07f26ad2e81a52c937e4c9ee437 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 13:02:13 +0000 Subject: [PATCH 28/80] fixup test_loc --- pandas/tests/indexing/test_loc.py | 35 ++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/pandas/tests/indexing/test_loc.py b/pandas/tests/indexing/test_loc.py index 61e95de3caf0d..3f9971f8b9c7b 100644 --- a/pandas/tests/indexing/test_loc.py +++ b/pandas/tests/indexing/test_loc.py @@ -382,7 +382,10 @@ def test_loc_setitem_slice(self): df2 = DataFrame({"a": [0, 1, 1], "b": [100, 200, 300]}, dtype="uint64") ix = df1["a"] == 1 newb2 = df2.loc[ix, "b"] - df1.loc[ix, "b"] = newb2 + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df1.loc[ix, "b"] = newb2 expected = DataFrame({"a": [0, 1, 1], "b": [100, 200, 300]}, dtype="uint64") tm.assert_frame_equal(df2, expected) @@ -869,7 +872,10 @@ def test_loc_setitem_with_scalar_index(self, indexer, value): # elementwisely, not using "setter('A', ['Z'])". df = DataFrame([[1, 2], [3, 4]], columns=["A", "B"]) - df.loc[0, indexer] = value + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df.loc[0, indexer] = value result = df.loc[0, "A"] assert is_scalar(result) and result == "Z" @@ -1415,8 +1421,11 @@ def test_loc_setitem_categorical_values_partial_column_slice(self): # the Categorical df = DataFrame({"a": [1, 1, 1, 1, 1], "b": list("aaaaa")}) exp = DataFrame({"a": [1, "b", "b", 1, 1], "b": list("aabba")}) - df.loc[1:2, "a"] = Categorical(["b", "b"], categories=["a", "b"]) - df.loc[2:3, "b"] = Categorical(["b", "b"], categories=["a", "b"]) + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df.loc[1:2, "a"] = Categorical(["b", "b"], categories=["a", "b"]) + df.loc[2:3, "b"] = Categorical(["b", "b"], categories=["a", "b"]) tm.assert_frame_equal(df, exp) def test_loc_setitem_single_row_categorical(self): @@ -1537,11 +1546,17 @@ def test_loc_setitem_2d_to_1d_raises(self): ] ) with pytest.raises(ValueError, match=msg): - ser.loc[range(2)] = data + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + ser.loc[range(2)] = data msg = r"could not broadcast input array from shape \(2,2\) into shape \(2,?\)" with pytest.raises(ValueError, match=msg): - ser.loc[:] = data + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + ser.loc[:] = data def test_loc_getitem_interval_index(self): # GH#19977 @@ -1607,7 +1622,10 @@ def test_loc_setitem_cast2(self): # dtype conversion on setting df = DataFrame(np.random.rand(30, 3), columns=tuple("ABC")) df["event"] = np.nan - df.loc[10, "event"] = "foo" + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df.loc[10, "event"] = "foo" result = df.dtypes expected = Series( [np.dtype("float64")] * 3 + [np.dtype("object")], @@ -2910,7 +2928,8 @@ def test_loc_setitem_uint8_upcast(value): # GH#26049 df = DataFrame([1, 2, 3, 4], columns=["col1"], dtype="uint8") - df.loc[2, "col1"] = value # value that can't be held in uint8 + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + df.loc[2, "col1"] = value # value that can't be held in uint8 expected = DataFrame([1, 2, 300, 4], columns=["col1"], dtype="uint16") tm.assert_frame_equal(df, expected) From 3aed02f2cfb1178a11c4ebe164727f76446620cc Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 13:02:53 +0000 Subject: [PATCH 29/80] fixup test_iloc --- pandas/tests/indexing/test_iloc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexing/test_iloc.py b/pandas/tests/indexing/test_iloc.py index 27dd16172b992..974ac38354da1 100644 --- a/pandas/tests/indexing/test_iloc.py +++ b/pandas/tests/indexing/test_iloc.py @@ -723,7 +723,10 @@ def test_iloc_setitem_with_scalar_index(self, indexer, value): # elementwisely, not using "setter('A', ['Z'])". df = DataFrame([[1, 2], [3, 4]], columns=["A", "B"]) - df.iloc[0, indexer] = value + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df.iloc[0, indexer] = value result = df.iloc[0, 0] assert is_scalar(result) and result == "Z" From 25f0693b958d739852304287b6ce5f6f5fb6bac3 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 13:03:18 +0000 Subject: [PATCH 30/80] fixup test_at --- pandas/tests/indexing/test_at.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexing/test_at.py b/pandas/tests/indexing/test_at.py index 9df6cb640257e..2934c73b74a98 100644 --- a/pandas/tests/indexing/test_at.py +++ b/pandas/tests/indexing/test_at.py @@ -122,7 +122,10 @@ def test_at_datetime_index(self, row): index=DatetimeIndex(data=["2019-01-01", "2019-01-02"]), ) - df.at[row, 0] = 0.5 + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df.at[row, 0] = 0.5 tm.assert_frame_equal(df, expected) From 0e5fb73dee0dd5c522fabafafcf5f8b2c278c5ff Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 13:30:05 +0000 Subject: [PATCH 31/80] fixup test_groupby --- pandas/tests/groupby/test_groupby.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 3baf2d86010f7..820645e262e0e 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -132,7 +132,9 @@ def f_1(grp): result = df.groupby("A").apply(f_1)[["B"]] e = expected.copy() - e.loc["Tiger"] = np.nan + with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): + # TODO is this right? Should it warn when setting new column? + e.loc["Tiger"] = np.nan tm.assert_frame_equal(result, e) def f_2(grp): @@ -141,7 +143,8 @@ def f_2(grp): return grp.iloc[0] result = df.groupby("A").apply(f_2)[["B"]] - e = expected.copy() + # Explicit cast to float to avoid implicit cast when setting nan + e = expected.copy().astype({"B": "float"}) e.loc["Pony"] = np.nan tm.assert_frame_equal(result, e) From 9b9e9757879bfd17158d46d47c0dff4e32ff9ec7 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 13 Jan 2023 16:31:43 +0000 Subject: [PATCH 32/80] fixup some doc warnings --- doc/source/user_guide/categorical.rst | 1 + doc/source/user_guide/cookbook.rst | 2 +- doc/source/user_guide/missing_data.rst | 2 +- doc/source/whatsnew/v0.15.0.rst | 2 +- doc/source/whatsnew/v0.17.0.rst | 2 +- doc/source/whatsnew/v0.21.0.rst | 1 + doc/source/whatsnew/v1.3.0.rst | 1 + pandas/tests/groupby/test_groupby.py | 1 - 8 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/source/user_guide/categorical.rst b/doc/source/user_guide/categorical.rst index f3d68f4c471c1..c42daf3a94a46 100644 --- a/doc/source/user_guide/categorical.rst +++ b/doc/source/user_guide/categorical.rst @@ -787,6 +787,7 @@ Setting values by assigning categorical data will also check that the ``categori Assigning a ``Categorical`` to parts of a column of other types will use the values: .. ipython:: python + :okwarning: df = pd.DataFrame({"a": [1, 1, 1, 1, 1], "b": ["a", "a", "a", "a", "a"]}) df.loc[1:2, "a"] = pd.Categorical(["b", "b"], categories=["a", "b"]) diff --git a/doc/source/user_guide/cookbook.rst b/doc/source/user_guide/cookbook.rst index daf5a0e481b8e..c0a31ae0123c7 100644 --- a/doc/source/user_guide/cookbook.rst +++ b/doc/source/user_guide/cookbook.rst @@ -125,7 +125,7 @@ Building criteria .. ipython:: python - df.loc[(df["BBB"] > 25) | (df["CCC"] >= 75), "AAA"] = 0.1 + df.loc[(df["BBB"] > 25) | (df["CCC"] >= 75), "AAA"] = 999 df `Select rows with data closest to certain value using argsort diff --git a/doc/source/user_guide/missing_data.rst b/doc/source/user_guide/missing_data.rst index aefb5f0d3d2df..8caa6ee3efb03 100644 --- a/doc/source/user_guide/missing_data.rst +++ b/doc/source/user_guide/missing_data.rst @@ -123,7 +123,7 @@ the missing value type chosen: .. ipython:: python - s = pd.Series([1, 2, 3]) + s = pd.Series([1., 2., 3.]) s.loc[0] = None s diff --git a/doc/source/whatsnew/v0.15.0.rst b/doc/source/whatsnew/v0.15.0.rst index f52253687ecfd..6047ef454548d 100644 --- a/doc/source/whatsnew/v0.15.0.rst +++ b/doc/source/whatsnew/v0.15.0.rst @@ -748,7 +748,7 @@ Other notable API changes: .. ipython:: python - s = pd.Series([1, 2, 3]) + s = pd.Series([1., 2., 3.]) s.loc[0] = None s diff --git a/doc/source/whatsnew/v0.17.0.rst b/doc/source/whatsnew/v0.17.0.rst index 7067407604d24..abbda2ffc9be2 100644 --- a/doc/source/whatsnew/v0.17.0.rst +++ b/doc/source/whatsnew/v0.17.0.rst @@ -738,7 +738,7 @@ Boolean comparisons of a ``Series`` vs ``None`` will now be equivalent to compar .. ipython:: python - s = pd.Series(range(3)) + s = pd.Series(range(3), dtype="float") s.iloc[1] = None s diff --git a/doc/source/whatsnew/v0.21.0.rst b/doc/source/whatsnew/v0.21.0.rst index 1bbbbdc7e5410..4048dd248830e 100644 --- a/doc/source/whatsnew/v0.21.0.rst +++ b/doc/source/whatsnew/v0.21.0.rst @@ -756,6 +756,7 @@ Previously assignments, ``.where()`` and ``.fillna()`` with a ``bool`` assignmen New behavior .. ipython:: python + :okwarning: s[1] = True s diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 537463d293287..94ba218629991 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -502,6 +502,7 @@ Setting non-boolean values into a :class:`Series` with ``dtype=bool`` now consis casts to ``dtype=object`` (:issue:`38709`) .. ipython:: python + :okwarning: orig = pd.Series([True, False]) ser = orig.copy() diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 820645e262e0e..1d743d90c511a 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -133,7 +133,6 @@ def f_1(grp): result = df.groupby("A").apply(f_1)[["B"]] e = expected.copy() with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): - # TODO is this right? Should it warn when setting new column? e.loc["Tiger"] = np.nan tm.assert_frame_equal(result, e) From 741b37d7e9e465878541869cae588394973aab1d Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 17 Mar 2023 09:52:38 +0000 Subject: [PATCH 33/80] post-merge fixup --- pandas/core/internals/blocks.py | 3 ++- pandas/tests/copy_view/test_indexing.py | 13 +++++++++++-- pandas/tests/groupby/test_value_counts.py | 1 - 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 60392623a2ca6..0b981528de073 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1050,7 +1050,8 @@ def setitem(self, indexer, value, using_cow: bool = False) -> Block: warnings.warn( f"Setting an item of incompatible dtype is deprecated " "and will raise in a future error of pandas. " - f"Value {value} has dtype incompatible with {values.dtype}", + f"Value '{value}' has dtype incompatible with {values.dtype}, " + "please explicitly cast to a compatible dtype first.", FutureWarning, stacklevel=find_stack_level(), ) diff --git a/pandas/tests/copy_view/test_indexing.py b/pandas/tests/copy_view/test_indexing.py index 942e79feba888..781779ffd2004 100644 --- a/pandas/tests/copy_view/test_indexing.py +++ b/pandas/tests/copy_view/test_indexing.py @@ -1021,7 +1021,10 @@ def test_dataframe_add_column_from_series(backend): ], ) def test_set_value_copy_only_necessary_column( - using_copy_on_write, indexer_func, indexer, val + using_copy_on_write, + indexer_func, + indexer, + val, ): # When setting inplace, only copy column that is modified instead of the whole # block (by splitting the block) @@ -1030,7 +1033,13 @@ def test_set_value_copy_only_necessary_column( df_orig = df.copy() view = df[:] - indexer_func(df)[indexer] = val + if val == "a" and indexer[0] != slice(None): + with tm.assert_produces_warning( + FutureWarning, match="Setting an item of incompatible dtype is deprecated" + ): + indexer_func(df)[indexer] = val + else: + indexer_func(df)[indexer] = val if using_copy_on_write: assert np.shares_memory(get_array(df, "b"), get_array(view, "b")) diff --git a/pandas/tests/groupby/test_value_counts.py b/pandas/tests/groupby/test_value_counts.py index a288bb29516d4..5477ad75a56f7 100644 --- a/pandas/tests/groupby/test_value_counts.py +++ b/pandas/tests/groupby/test_value_counts.py @@ -55,7 +55,6 @@ def seed_df(seed_nans, n, m): "3rd": np.random.randint(1, m + 1, n), } ) - frame["3rd"] = frame["3rd"].astype("float64") if seed_nans: # Explicitly cast to float to avoid implicit cast when setting nan From c536d8aa2041f3a7b194ca4e5643a3487bd3790a Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 17 Mar 2023 11:06:21 +0000 Subject: [PATCH 34/80] change dtype in doctest --- pandas/core/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 060197e337f41..5b14248e5018f 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -7835,7 +7835,7 @@ def asof(self, where, subset=None): Take all columns into consideration - >>> df = pd.DataFrame({'a': [10, 20, 30, 40, 50], + >>> df = pd.DataFrame({'a': [10., 20., 30., 40., 50.], ... 'b': [None, None, None, None, 500]}, ... index=pd.DatetimeIndex(['2018-02-27 09:01:00', ... '2018-02-27 09:02:00', From 142992e8e14b95df45237f690458eaf4d2c1d7b0 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 17 Mar 2023 11:58:19 +0000 Subject: [PATCH 35/80] fixup doctest --- pandas/core/generic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 5b14248e5018f..9b7d468b2613e 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -7853,9 +7853,9 @@ def asof(self, where, subset=None): >>> df.asof(pd.DatetimeIndex(['2018-02-27 09:03:30', ... '2018-02-27 09:04:30']), ... subset=['a']) - a b - 2018-02-27 09:03:30 30 NaN - 2018-02-27 09:04:30 40 NaN + a b + 2018-02-27 09:03:30 30.0 NaN + 2018-02-27 09:04:30 40.0 NaN """ if isinstance(where, str): where = Timestamp(where) From 45518557d19596b37e09f63a0ea38b24992aa077 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 17 Mar 2023 13:59:59 +0000 Subject: [PATCH 36/80] explicit cast in test --- pandas/tests/groupby/test_nunique.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/groupby/test_nunique.py b/pandas/tests/groupby/test_nunique.py index 282c91c82f5b1..661003d081bda 100644 --- a/pandas/tests/groupby/test_nunique.py +++ b/pandas/tests/groupby/test_nunique.py @@ -51,6 +51,7 @@ def check_nunique(df, keys, as_index=True): check_nunique(frame, ["jim"]) check_nunique(frame, ["jim", "joe"]) + frame = frame.astype({"julie": float}) # Explicit cast to avoid implicit cast below frame.loc[1::17, "jim"] = None frame.loc[3::37, "joe"] = None frame.loc[7::19, "julie"] = None From 01b0c7289564c96d703876e7602ce372253c9e46 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 20 Mar 2023 17:24:04 +0000 Subject: [PATCH 37/80] fixup test for COW --- pandas/tests/copy_view/test_indexing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/copy_view/test_indexing.py b/pandas/tests/copy_view/test_indexing.py index 781779ffd2004..193d2f0d793e8 100644 --- a/pandas/tests/copy_view/test_indexing.py +++ b/pandas/tests/copy_view/test_indexing.py @@ -914,7 +914,8 @@ def test_column_as_series_set_with_upcast( s[0] = "foo" expected = Series([1, 2, 3], name="a") elif using_copy_on_write or using_array_manager: - s[0] = "foo" + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + s[0] = "foo" expected = Series(["foo", 2, 3], dtype=object, name="a") else: with pd.option_context("chained_assignment", "warn"): From df740e6d6fe57e2ef97d0348b3751b9cbf815edf Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 14 Apr 2023 18:49:38 +0100 Subject: [PATCH 38/80] fixup COW --- pandas/_testing/contexts.py | 6 +++--- pandas/tests/indexing/test_chaining_and_caching.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/_testing/contexts.py b/pandas/_testing/contexts.py index fb5b7b967f6bf..7fabd467295f6 100644 --- a/pandas/_testing/contexts.py +++ b/pandas/_testing/contexts.py @@ -205,8 +205,8 @@ def use_numexpr(use, min_elements=None) -> Generator[None, None, None]: set_option("compute.use_numexpr", olduse) -def raises_chained_assignment_error(): - if PYPY: +def raises_chained_assignment_error(extra_warnings=()): + if PYPY and extra_warnings is None: from contextlib import nullcontext return nullcontext() @@ -214,7 +214,7 @@ def raises_chained_assignment_error(): from pandas._testing import assert_produces_warning return assert_produces_warning( - ChainedAssignmentError, + (ChainedAssignmentError, *extra_warnings), match=( "A value is trying to be set on a copy of a DataFrame or Series " "through chained assignment" diff --git a/pandas/tests/indexing/test_chaining_and_caching.py b/pandas/tests/indexing/test_chaining_and_caching.py index d2224988b70fc..6488ff2e64ea7 100644 --- a/pandas/tests/indexing/test_chaining_and_caching.py +++ b/pandas/tests/indexing/test_chaining_and_caching.py @@ -457,7 +457,7 @@ def test_detect_chained_assignment_changing_dtype( df.loc[2]["D"] = "foo" with tm.raises_chained_assignment_error(): df.loc[2]["C"] = "foo" - with tm.raises_chained_assignment_error(): + with tm.raises_chained_assignment_error(extra_warnings=(FutureWarning,)): df["C"][2] = "foo" tm.assert_frame_equal(df, df_original) From 8386032d29aab48986be3d1cc4fb0b4396ed44be Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 24 Apr 2023 15:36:06 +0100 Subject: [PATCH 39/80] catch warnings in testsetitemcastingequivalents --- pandas/tests/series/indexing/test_setitem.py | 57 +++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 764e4af268e71..4ea1df567b90f 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -746,41 +746,49 @@ def _check_inplace(self, is_inplace, orig, arr, obj): # otherwise original array should be unchanged tm.assert_equal(arr, orig._values) - @pytest.mark.filterwarnings( - "ignore:Setting an item of incompatible dtype:FutureWarning" - ) - def test_int_key(self, obj, key, expected, val, indexer_sli, is_inplace): + # @pytest.mark.filterwarnings( + # "ignore:Setting an item of incompatible dtype:FutureWarning" + # ) + def test_int_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): if not isinstance(key, int): return - self.check_indexer(obj, key, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, key, expected, val, indexer_sli, is_inplace) if indexer_sli is tm.loc: - self.check_indexer(obj, key, expected, val, tm.at, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, key, expected, val, tm.at, is_inplace) elif indexer_sli is tm.iloc: - self.check_indexer(obj, key, expected, val, tm.iat, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, key, expected, val, tm.iat, is_inplace) rng = range(key, key + 1) - self.check_indexer(obj, rng, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, rng, expected, val, indexer_sli, is_inplace) if indexer_sli is not tm.loc: # Note: no .loc because that handles slice edges differently slc = slice(key, key + 1) - self.check_indexer(obj, slc, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, slc, expected, val, indexer_sli, is_inplace) ilkey = [key] - self.check_indexer(obj, ilkey, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, ilkey, expected, val, indexer_sli, is_inplace) indkey = np.array(ilkey) - self.check_indexer(obj, indkey, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, indkey, expected, val, indexer_sli, is_inplace) genkey = (x for x in [key]) - self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) @pytest.mark.filterwarnings( "ignore:Setting an item of incompatible dtype:FutureWarning" ) - def test_slice_key(self, obj, key, expected, val, indexer_sli, is_inplace): + def test_slice_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): if not isinstance(key, slice): return @@ -800,7 +808,7 @@ def test_slice_key(self, obj, key, expected, val, indexer_sli, is_inplace): @pytest.mark.filterwarnings( "ignore:Setting an item of incompatible dtype:FutureWarning" ) - def test_mask_key(self, obj, key, expected, val, indexer_sli): + def test_mask_key(self, obj, key, expected, warn, val, indexer_sli): # setitem with boolean mask mask = np.zeros(obj.shape, dtype=bool) mask[key] = True @@ -816,7 +824,7 @@ def test_mask_key(self, obj, key, expected, val, indexer_sli): indexer_sli(obj)[mask] = val tm.assert_series_equal(obj, expected) - def test_series_where(self, obj, key, expected, val, is_inplace): + def test_series_where(self, obj, key, expected, warn, val, is_inplace): mask = np.zeros(obj.shape, dtype=bool) mask[key] = True @@ -836,7 +844,7 @@ def test_series_where(self, obj, key, expected, val, is_inplace): self._check_inplace(is_inplace, orig, arr, obj) - def test_index_where(self, obj, key, expected, val): + def test_index_where(self, obj, key, expected, warn, val): mask = np.zeros(obj.shape, dtype=bool) mask[key] = True @@ -844,7 +852,7 @@ def test_index_where(self, obj, key, expected, val): expected_idx = Index(expected, dtype=expected.dtype) tm.assert_index_equal(res, expected_idx) - def test_index_putmask(self, obj, key, expected, val): + def test_index_putmask(self, obj, key, expected, warn, val): mask = np.zeros(obj.shape, dtype=bool) mask[key] = True @@ -853,7 +861,7 @@ def test_index_putmask(self, obj, key, expected, val): @pytest.mark.parametrize( - "obj,expected,key", + "obj,expected,key,warn", [ pytest.param( # GH#45568 setting a valid NA value into IntervalDtype[int] should @@ -864,6 +872,7 @@ def test_index_putmask(self, obj, key, expected, val): dtype="interval[float64]", ), 1, + None, id="interval_int_na_value", ), pytest.param( @@ -871,12 +880,14 @@ def test_index_putmask(self, obj, key, expected, val): Series([2, 3, 4, 5, 6, 7, 8, 9, 10]), Series([np.nan, 3, np.nan, 5, np.nan, 7, np.nan, 9, np.nan]), slice(None, None, 2), + None, id="int_series_slice_key_step", ), pytest.param( Series([True, True, False, False]), Series([np.nan, True, np.nan, False], dtype=object), slice(None, None, 2), + None, id="bool_series_slice_key_step", ), pytest.param( @@ -884,6 +895,7 @@ def test_index_putmask(self, obj, key, expected, val): Series(np.arange(10)), Series([np.nan, np.nan, np.nan, np.nan, np.nan, 5, 6, 7, 8, 9]), slice(None, 5), + None, id="int_series_slice_key", ), pytest.param( @@ -891,6 +903,7 @@ def test_index_putmask(self, obj, key, expected, val): Series([1, 2, 3]), Series([np.nan, 2, 3]), 0, + FutureWarning, id="int_series_int_key", ), pytest.param( @@ -899,6 +912,7 @@ def test_index_putmask(self, obj, key, expected, val): Series([np.nan], dtype=object), # TODO: maybe go to float64 since we are changing the _whole_ Series? 0, + FutureWarning, id="bool_series_int_key_change_all", ), pytest.param( @@ -906,6 +920,7 @@ def test_index_putmask(self, obj, key, expected, val): Series([False, True]), Series([np.nan, True], dtype=object), 0, + FutureWarning, id="bool_series_int_key", ), ], @@ -954,6 +969,10 @@ def expected(self, dtype): def key(self): return 0 + @pytest.fixture + def warn(self): + return FutureWarning + class TestSetitemDT64IntoInt(SetitemCastingEquivalents): # GH#39619 dont cast dt64 to int when doing this setitem @@ -1350,7 +1369,7 @@ class TestCoercionFloat32(CoercionTest): def obj(self): return Series([1.1, 2.2, 3.3, 4.4], dtype=np.float32) - def test_slice_key(self, obj, key, expected, val, indexer_sli, is_inplace): + def test_slice_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): super().test_slice_key(obj, key, expected, val, indexer_sli, is_inplace) if type(val) is float: From 39decb233eaed30204169602d4cb87756cb97462 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 24 Apr 2023 15:44:13 +0100 Subject: [PATCH 40/80] wip --- pandas/tests/series/indexing/test_setitem.py | 61 +++++++++++++++++--- 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 4ea1df567b90f..ee724126a5bf2 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -1009,6 +1009,10 @@ def val(self, scalar, request): return scalar return box([scalar, scalar]) + @pytest.fixture + def warn(self): + return FutureWarning + class TestSetitemNAPeriodDtype(SetitemCastingEquivalents): # Setting compatible NA values into Series with PeriodDtype @@ -1032,6 +1036,10 @@ def key(self, request): def val(self, request): return request.param + @pytest.fixture + def warn(self): + return None + class TestSetitemNADatetimeLikeDtype(SetitemCastingEquivalents): # some nat-like values should be cast to datetime64/timedelta64 when @@ -1081,6 +1089,10 @@ def expected(self, obj, val, is_inplace): def key(self): return 0 + @pytest.fixture + def warn(self): + return None + class TestSetitemMismatchedTZCastsToObject(SetitemCastingEquivalents): # GH#24024 @@ -1109,20 +1121,25 @@ def expected(self, obj, val): ) return expected + @pytest.fixture + def warn(self): + return None + @pytest.mark.parametrize( - "obj,expected", + "obj,expected,warn", [ # For numeric series, we should coerce to NaN. - (Series([1, 2, 3]), Series([np.nan, 2, 3])), - (Series([1.0, 2.0, 3.0]), Series([np.nan, 2.0, 3.0])), + (Series([1, 2, 3]), Series([np.nan, 2, 3]), FutureWarning), + (Series([1.0, 2.0, 3.0]), Series([np.nan, 2.0, 3.0]), None), # For datetime series, we should coerce to NaT. ( Series([datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)]), Series([NaT, datetime(2000, 1, 2), datetime(2000, 1, 3)]), + None, ), # For objects, we should preserve the None value. - (Series(["foo", "bar", "baz"]), Series([None, "bar", "baz"])), + (Series(["foo", "bar", "baz"]), Series([None, "bar", "baz"]), None), ], ) class TestSeriesNoneCoercion(SetitemCastingEquivalents): @@ -1166,6 +1183,10 @@ def expected(self, obj, val): idx = IntervalIndex(data, dtype="Interval[float64]") return Series(idx) + @pytest.fixture + def warn(self): + return None + class TestSetitemRangeIntoIntegerSeries(SetitemCastingEquivalents): # GH#44261 Setting a range with sufficiently-small integers into @@ -1191,6 +1212,10 @@ def expected(self, any_int_numpy_dtype): exp = Series([2, 3, 2, 3, 4], dtype=dtype) return exp + @pytest.fixture + def warn(self): + return None + @pytest.mark.parametrize( "val", @@ -1220,6 +1245,10 @@ def expected(self, val): res_values[:2] = val return Series(res_values) + @pytest.fixture + def warn(self): + return FutureWarning + @pytest.mark.parametrize("val", [512, np.int16(512)]) class TestSetitemIntoIntegerSeriesNeedsUpcast(SetitemCastingEquivalents): @@ -1235,6 +1264,10 @@ def key(self): def expected(self): return Series([1, 512, 3], dtype=np.int16) + @pytest.fixture + def warn(self): + return FutureWarning + @pytest.mark.parametrize("val", [2**33 + 1.0, 2**33 + 1.1, 2**62]) class TestSmallIntegerSetitemUpcast(SetitemCastingEquivalents): @@ -1255,6 +1288,10 @@ def expected(self, val): dtype = "i8" return Series([val, 2, 3], dtype=dtype) + @pytest.fixture + def warn(self): + return FutureWarning + class CoercionTest(SetitemCastingEquivalents): # Tests ported from tests.indexing.test_coercion @@ -1271,7 +1308,8 @@ def expected(self, obj, key, val, exp_dtype): @pytest.mark.parametrize( - "val,exp_dtype", [(np.int32(1), np.int8), (np.int16(2**9), np.int16)] + "val,exp_dtype,warn", + [(np.int32(1), np.int8, None), (np.int16(2**9), np.int16, FutureWarning)], ) class TestCoercionInt8(CoercionTest): # previously test_setitem_series_int8 in tests.indexing.test_coercion @@ -1288,10 +1326,19 @@ class TestCoercionObject(CoercionTest): def obj(self): return Series(["a", "b", "c", "d"], dtype=object) + @pytest.fixture + def warn(self): + return None + @pytest.mark.parametrize( - "val,exp_dtype", - [(1, np.complex128), (1.1, np.complex128), (1 + 1j, np.complex128), (True, object)], + "val,exp_dtype,warn", + [ + (1, np.complex128, None), + (1.1, np.complex128, None), + (1 + 1j, np.complex128, None), + (True, object, FutureWarning), + ], ) class TestCoercionComplex(CoercionTest): # previously test_setitem_series_complex128 in tests.indexing.test_coercion From 75498885c18e87039a5d4377828f70a3cecd2e34 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 24 Apr 2023 16:13:30 +0100 Subject: [PATCH 41/80] fixup setitem test int key! --- pandas/tests/series/indexing/test_setitem.py | 72 +++++++++++++------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index ee724126a5bf2..de4c61ba03664 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -746,9 +746,6 @@ def _check_inplace(self, is_inplace, orig, arr, obj): # otherwise original array should be unchanged tm.assert_equal(arr, orig._values) - # @pytest.mark.filterwarnings( - # "ignore:Setting an item of incompatible dtype:FutureWarning" - # ) def test_int_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): if not isinstance(key, int): return @@ -1348,14 +1345,14 @@ def obj(self): @pytest.mark.parametrize( - "val,exp_dtype", + "val,exp_dtype,warn", [ - (1, object), - ("3", object), - (3, object), - (1.1, object), - (1 + 1j, object), - (True, bool), + (1, object, FutureWarning), + ("3", object, FutureWarning), + (3, object, FutureWarning), + (1.1, object, FutureWarning), + (1 + 1j, object, FutureWarning), + (True, bool, None), ], ) class TestCoercionBool(CoercionTest): @@ -1366,8 +1363,13 @@ def obj(self): @pytest.mark.parametrize( - "val,exp_dtype", - [(1, np.int64), (1.1, np.float64), (1 + 1j, np.complex128), (True, object)], + "val,exp_dtype,warn", + [ + (1, np.int64, None), + (1.1, np.float64, FutureWarning), + (1 + 1j, np.complex128, FutureWarning), + (True, object, FutureWarning), + ], ) class TestCoercionInt64(CoercionTest): # previously test_setitem_series_int64 in tests.indexing.test_coercion @@ -1377,8 +1379,13 @@ def obj(self): @pytest.mark.parametrize( - "val,exp_dtype", - [(1, np.float64), (1.1, np.float64), (1 + 1j, np.complex128), (True, object)], + "val,exp_dtype,warn", + [ + (1, np.float64, None), + (1.1, np.float64, None), + (1 + 1j, np.complex128, FutureWarning), + (True, object, FutureWarning), + ], ) class TestCoercionFloat64(CoercionTest): # previously test_setitem_series_float64 in tests.indexing.test_coercion @@ -1388,27 +1395,28 @@ def obj(self): @pytest.mark.parametrize( - "val,exp_dtype", + "val,exp_dtype,warn", [ - (1, np.float32), + (1, np.float32, None), pytest.param( 1.1, np.float32, + None, marks=pytest.mark.xfail( reason="np.float32(1.1) ends up as 1.100000023841858, so " "np_can_hold_element raises and we cast to float64", ), ), - (1 + 1j, np.complex128), - (True, object), - (np.uint8(2), np.float32), - (np.uint32(2), np.float32), + (1 + 1j, np.complex128, FutureWarning), + (True, object, FutureWarning), + (np.uint8(2), np.float32, None), + (np.uint32(2), np.float32, None), # float32 cannot hold np.iinfo(np.uint32).max exactly # (closest it can hold is 4294967300.0 which off by 5.0), so # we cast to float64 - (np.uint32(np.iinfo(np.uint32).max), np.float64), - (np.uint64(2), np.float32), - (np.int64(2), np.float32), + (np.uint32(np.iinfo(np.uint32).max), np.float64, FutureWarning), + (np.uint64(2), np.float32, None), + (np.int64(2), np.float32, None), ], ) class TestCoercionFloat32(CoercionTest): @@ -1417,7 +1425,7 @@ def obj(self): return Series([1.1, 2.2, 3.3, 4.4], dtype=np.float32) def test_slice_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): - super().test_slice_key(obj, key, expected, val, indexer_sli, is_inplace) + super().test_slice_key(obj, key, expected, warn, val, indexer_sli, is_inplace) if type(val) is float: # the xfail would xpass bc test_slice_key short-circuits @@ -1435,6 +1443,10 @@ class TestCoercionDatetime64(CoercionTest): def obj(self): return Series(date_range("2011-01-01", freq="D", periods=4)) + @pytest.fixture + def warn(self): + return None + @pytest.mark.parametrize( "val,exp_dtype", @@ -1453,6 +1465,10 @@ def obj(self): tz = "US/Eastern" return Series(date_range("2011-01-01", freq="D", periods=4, tz=tz)) + @pytest.fixture + def warn(self): + return None + @pytest.mark.parametrize( "val,exp_dtype", @@ -1464,6 +1480,10 @@ class TestCoercionTimedelta64(CoercionTest): def obj(self): return Series(timedelta_range("1 day", periods=4)) + @pytest.fixture + def warn(self): + return None + @pytest.mark.parametrize( "val", ["foo", Period("2016", freq="Y"), Interval(1, 2, closed="both")] @@ -1480,6 +1500,10 @@ class TestPeriodIntervalCoercion(CoercionTest): def obj(self, request): return Series(request.param) + @pytest.fixture + def warn(self): + return None + def test_20643(): # closed by GH#45121 From 527fa2deb19ba756e055b564b0694b460c486966 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 24 Apr 2023 16:24:54 +0100 Subject: [PATCH 42/80] getting there! --- pandas/tests/series/indexing/test_setitem.py | 41 ++++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index de4c61ba03664..fbeecf3953264 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -782,29 +782,29 @@ def test_int_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): with tm.assert_produces_warning(warn): self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) - @pytest.mark.filterwarnings( - "ignore:Setting an item of incompatible dtype:FutureWarning" - ) def test_slice_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): if not isinstance(key, slice): return + # if isinstance(key, slice) and key.start is None and key.stop is None: + # warn = None if indexer_sli is not tm.loc: # Note: no .loc because that handles slice edges differently - self.check_indexer(obj, key, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, key, expected, val, indexer_sli, is_inplace) ilkey = list(range(len(obj)))[key] - self.check_indexer(obj, ilkey, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, ilkey, expected, val, indexer_sli, is_inplace) indkey = np.array(ilkey) - self.check_indexer(obj, indkey, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, indkey, expected, val, indexer_sli, is_inplace) genkey = (x for x in indkey) - self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) + with tm.assert_produces_warning(warn): + self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) - @pytest.mark.filterwarnings( - "ignore:Setting an item of incompatible dtype:FutureWarning" - ) def test_mask_key(self, obj, key, expected, warn, val, indexer_sli): # setitem with boolean mask mask = np.zeros(obj.shape, dtype=bool) @@ -877,14 +877,14 @@ def test_index_putmask(self, obj, key, expected, warn, val): Series([2, 3, 4, 5, 6, 7, 8, 9, 10]), Series([np.nan, 3, np.nan, 5, np.nan, 7, np.nan, 9, np.nan]), slice(None, None, 2), - None, + FutureWarning, id="int_series_slice_key_step", ), pytest.param( Series([True, True, False, False]), Series([np.nan, True, np.nan, False], dtype=object), slice(None, None, 2), - None, + FutureWarning, id="bool_series_slice_key_step", ), pytest.param( @@ -892,7 +892,7 @@ def test_index_putmask(self, obj, key, expected, warn, val): Series(np.arange(10)), Series([np.nan, np.nan, np.nan, np.nan, np.nan, 5, 6, 7, 8, 9]), slice(None, 5), - None, + FutureWarning, id="int_series_slice_key", ), pytest.param( @@ -1215,11 +1215,14 @@ def warn(self): @pytest.mark.parametrize( - "val", + "val, warn", [ - np.array([2.0, 3.0]), - np.array([2.5, 3.5]), - np.array([2**65, 2**65 + 1], dtype=np.float64), # all ints, but can't cast + (np.array([2.0, 3.0]), None), + (np.array([2.5, 3.5]), FutureWarning), + ( + np.array([2**65, 2**65 + 1], dtype=np.float64), + FutureWarning, + ), # all ints, but can't cast ], ) class TestSetitemFloatNDarrayIntoIntegerSeries(SetitemCastingEquivalents): @@ -1242,10 +1245,6 @@ def expected(self, val): res_values[:2] = val return Series(res_values) - @pytest.fixture - def warn(self): - return FutureWarning - @pytest.mark.parametrize("val", [512, np.int16(512)]) class TestSetitemIntoIntegerSeriesNeedsUpcast(SetitemCastingEquivalents): From caa35c3c5950d902a6f81cd01454ef5f8f5bf1a1 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 24 Apr 2023 16:37:29 +0100 Subject: [PATCH 43/80] fixup test_setitem --- pandas/core/internals/blocks.py | 8 ++++ pandas/tests/series/indexing/test_setitem.py | 39 +++++++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 9d5393ef041f8..b403135544915 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1123,6 +1123,14 @@ def putmask(self, mask, new, using_cow: bool = False) -> list[Block]: return [self.copy(deep=False)] return [self] except LossySetitemError: + warnings.warn( + f"Setting an item of incompatible dtype is deprecated " + "and will raise in a future error of pandas. " + f"Value '{new}' has dtype incompatible with {values.dtype}, " + "please explicitly cast to a compatible dtype first.", + FutureWarning, + stacklevel=find_stack_level(), + ) if self.ndim == 1 or self.shape[0] == 1: # no need to split columns diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index fbeecf3953264..1b49b29c69f69 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -251,7 +251,8 @@ def test_setitem_mask_cast(self): # GH#2746 # need to upcast ser = Series([1, 2], index=[1, 2], dtype="int64") - ser[[True, False]] = Series([0], index=[1], dtype="int64") + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[[True, False]] = Series([0], index=[1], dtype="int64") expected = Series([0, 2], index=[1, 2], dtype="int64") tm.assert_series_equal(ser, expected) @@ -409,12 +410,14 @@ def test_setitem_mask_smallint_no_upcast(self): mask = np.array([True, False, True]) ser = orig.copy() - ser[mask] = alt + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[mask] = alt expected = Series([245, 2, 246], dtype="uint8") tm.assert_series_equal(ser, expected) ser2 = orig.copy() - ser2.mask(mask, alt, inplace=True) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser2.mask(mask, alt, inplace=True) tm.assert_series_equal(ser2, expected) # FIXME: don't leave commented-out @@ -481,7 +484,8 @@ def test_setitem_callable_other(self): inc = lambda x: x + 1 ser = Series([1, 2, -1, 4]) - ser[ser < 0] = inc + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + ser[ser < 0] = inc expected = Series([1, 2, inc, 4]) tm.assert_series_equal(ser, expected) @@ -750,36 +754,36 @@ def test_int_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): if not isinstance(key, int): return - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, key, expected, val, indexer_sli, is_inplace) if indexer_sli is tm.loc: - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, key, expected, val, tm.at, is_inplace) elif indexer_sli is tm.iloc: - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, key, expected, val, tm.iat, is_inplace) rng = range(key, key + 1) - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, rng, expected, val, indexer_sli, is_inplace) if indexer_sli is not tm.loc: # Note: no .loc because that handles slice edges differently slc = slice(key, key + 1) - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, slc, expected, val, indexer_sli, is_inplace) ilkey = [key] - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, ilkey, expected, val, indexer_sli, is_inplace) indkey = np.array(ilkey) - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, indkey, expected, val, indexer_sli, is_inplace) genkey = (x for x in [key]) - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) def test_slice_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): @@ -790,19 +794,19 @@ def test_slice_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace) if indexer_sli is not tm.loc: # Note: no .loc because that handles slice edges differently - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, key, expected, val, indexer_sli, is_inplace) ilkey = list(range(len(obj)))[key] - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, ilkey, expected, val, indexer_sli, is_inplace) indkey = np.array(ilkey) - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, indkey, expected, val, indexer_sli, is_inplace) genkey = (x for x in indkey) - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): self.check_indexer(obj, genkey, expected, val, indexer_sli, is_inplace) def test_mask_key(self, obj, key, expected, warn, val, indexer_sli): @@ -818,7 +822,8 @@ def test_mask_key(self, obj, key, expected, warn, val, indexer_sli): indexer_sli(obj)[mask] = val return - indexer_sli(obj)[mask] = val + with tm.assert_produces_warning(warn, match="incompatible dtype"): + indexer_sli(obj)[mask] = val tm.assert_series_equal(obj, expected) def test_series_where(self, obj, key, expected, warn, val, is_inplace): From cc1542d3b3af247102deb98080389d130b4dd3b6 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Mon, 24 Apr 2023 16:47:41 +0100 Subject: [PATCH 44/80] getting there --- pandas/tests/frame/indexing/test_where.py | 21 ++++++++++++------- .../series/methods/test_convert_dtypes.py | 16 +++++++++++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/pandas/tests/frame/indexing/test_where.py b/pandas/tests/frame/indexing/test_where.py index a76daedc5ffb3..d4802bb26a062 100644 --- a/pandas/tests/frame/indexing/test_where.py +++ b/pandas/tests/frame/indexing/test_where.py @@ -191,14 +191,17 @@ def _check_set(df, cond, check_dtypes=True): return cond = df > 0 - _check_set(df, cond) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + _check_set(df, cond) cond = df >= 0 - _check_set(df, cond) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + _check_set(df, cond) # aligning cond = (df >= 0)[1:] - _check_set(df, cond) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + _check_set(df, cond) def test_where_series_slicing(self): # GH 10218 @@ -336,7 +339,8 @@ def test_where_bug_mixed(self, any_signed_int_numpy_dtype): tm.assert_frame_equal(result, expected) result = df.copy() - return_value = result.where(result > 2, np.nan, inplace=True) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + return_value = result.where(result > 2, np.nan, inplace=True) assert return_value is None tm.assert_frame_equal(result, expected) @@ -347,7 +351,8 @@ def test_where_bug_transposition(self): do_not_replace = b.isna() | (a > b) expected = a.copy() - expected[~do_not_replace] = b + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + expected[~do_not_replace] = b result = a.where(do_not_replace, b) tm.assert_frame_equal(result, expected) @@ -357,7 +362,8 @@ def test_where_bug_transposition(self): do_not_replace = b.isna() | (a > b) expected = a.copy() - expected[~do_not_replace] = b + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + expected[~do_not_replace] = b result = a.where(do_not_replace, b) tm.assert_frame_equal(result, expected) @@ -990,7 +996,8 @@ def test_where_dt64_2d(): # setting all of one column, none of the other expected = DataFrame({"A": other[:, 0], "B": dta[:, 1]}) - _check_where_equivalences(df, mask, other, expected) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + _check_where_equivalences(df, mask, other, expected) # setting part of one column, none of the other mask[1, 0] = True diff --git a/pandas/tests/series/methods/test_convert_dtypes.py b/pandas/tests/series/methods/test_convert_dtypes.py index d91cd6a43daea..f4523d19bdf16 100644 --- a/pandas/tests/series/methods/test_convert_dtypes.py +++ b/pandas/tests/series/methods/test_convert_dtypes.py @@ -193,7 +193,21 @@ def test_convert_dtypes( # Test that it is a copy copy = series.copy(deep=True) - result[result.notna()] = np.nan + if result.notna().sum() > 0 and result.dtype in [ + "int8", + "uint8", + "int16", + "uint16", + "int32", + "uint32", + "int64", + "uint64", + ]: + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + result[result.notna()] = np.nan + else: + with tm.assert_produces_warning(None): + result[result.notna()] = np.nan # Make sure original not changed tm.assert_series_equal(series, copy) From 492d443fcc17a451c2353144866f6d16f8948345 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Tue, 25 Apr 2023 16:54:26 +0100 Subject: [PATCH 45/80] fixup remaining warnings --- pandas/tests/frame/indexing/test_where.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pandas/tests/frame/indexing/test_where.py b/pandas/tests/frame/indexing/test_where.py index d4802bb26a062..a87a37a5a5bf5 100644 --- a/pandas/tests/frame/indexing/test_where.py +++ b/pandas/tests/frame/indexing/test_where.py @@ -191,16 +191,21 @@ def _check_set(df, cond, check_dtypes=True): return cond = df > 0 - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + if (df.dtypes == "int").any(): + warn = FutureWarning + else: + warn = None + + with tm.assert_produces_warning(warn, match="incompatible dtype"): _check_set(df, cond) cond = df >= 0 - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + with tm.assert_produces_warning(warn, match="incompatible dtype"): _check_set(df, cond) # aligning cond = (df >= 0)[1:] - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + with tm.assert_produces_warning(warn, match="incompatible dtype"): _check_set(df, cond) def test_where_series_slicing(self): @@ -996,8 +1001,7 @@ def test_where_dt64_2d(): # setting all of one column, none of the other expected = DataFrame({"A": other[:, 0], "B": dta[:, 1]}) - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - _check_where_equivalences(df, mask, other, expected) + _check_where_equivalences(df, mask, other, expected) # setting part of one column, none of the other mask[1, 0] = True @@ -1041,6 +1045,7 @@ def test_where_inplace_no_other(): # GH#51685 df = DataFrame({"a": [1, 2], "b": ["x", "y"]}) cond = DataFrame({"a": [True, False], "b": [False, True]}) - df.where(cond, inplace=True) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df.where(cond, inplace=True) expected = DataFrame({"a": [1, np.nan], "b": [np.nan, "y"]}) tm.assert_frame_equal(df, expected) From 90a64abeb2c629de23f045830d2c3fdfd6b05600 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Tue, 25 Apr 2023 17:02:21 +0100 Subject: [PATCH 46/80] fix test_update --- pandas/tests/series/methods/test_update.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pandas/tests/series/methods/test_update.py b/pandas/tests/series/methods/test_update.py index b06ed153bfe49..3ae8f233625a3 100644 --- a/pandas/tests/series/methods/test_update.py +++ b/pandas/tests/series/methods/test_update.py @@ -41,13 +41,13 @@ def test_update(self, using_copy_on_write): "other, dtype, expected, warn", [ # other is int - ([61, 63], "int32", Series([10, 61, 12], dtype="int32"), None), - ([61, 63], "int64", Series([10, 61, 12]), None), + ([61, 63], "int32", Series([10, 61, 12], dtype="int32"), FutureWarning), + ([61, 63], "int64", Series([10, 61, 12]), FutureWarning), ([61, 63], float, Series([10.0, 61.0, 12.0]), None), ([61, 63], object, Series([10, 61, 12], dtype=object), None), # other is float, but can be cast to int - ([61.0, 63.0], "int32", Series([10, 61, 12], dtype="int32"), None), - ([61.0, 63.0], "int64", Series([10, 61, 12]), None), + ([61.0, 63.0], "int32", Series([10, 61, 12], dtype="int32"), FutureWarning), + ([61.0, 63.0], "int64", Series([10, 61, 12]), FutureWarning), ([61.0, 63.0], float, Series([10.0, 61.0, 12.0]), None), ([61.0, 63.0], object, Series([10, 61.0, 12], dtype=object), None), # others is float, cannot be cast to int @@ -85,7 +85,8 @@ def test_update_dtypes(self, other, dtype, expected, warn): ) def test_update_from_non_series(self, series, other, expected): # GH 33215 - series.update(other) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + series.update(other) tm.assert_series_equal(series, expected) @pytest.mark.parametrize( From b3f6b931dd446384a759380146f638583ef8bbf7 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 26 Apr 2023 10:22:01 +0100 Subject: [PATCH 47/80] fixup some failing test --- pandas/tests/arrays/boolean/test_comparison.py | 15 ++++++++++++--- pandas/tests/copy_view/test_methods.py | 9 ++++++--- pandas/tests/frame/indexing/test_setitem.py | 3 ++- pandas/tests/frame/methods/test_map.py | 3 ++- pandas/tests/groupby/test_counting.py | 3 ++- pandas/tests/series/indexing/test_indexing.py | 3 ++- 6 files changed, 26 insertions(+), 10 deletions(-) diff --git a/pandas/tests/arrays/boolean/test_comparison.py b/pandas/tests/arrays/boolean/test_comparison.py index 2eeb9da574b1e..d265c43aa20f8 100644 --- a/pandas/tests/arrays/boolean/test_comparison.py +++ b/pandas/tests/arrays/boolean/test_comparison.py @@ -24,15 +24,24 @@ def dtype(): class TestComparisonOps(ComparisonOps): def test_compare_scalar(self, data, comparison_op): - self._compare_other(data, comparison_op, True) + with tm.assert_produces_warning( + FutureWarning, match="incompatible dtype", check_stacklevel=False + ): + self._compare_other(data, comparison_op, True) def test_compare_array(self, data, comparison_op): other = pd.array([True] * len(data), dtype="boolean") self._compare_other(data, comparison_op, other) other = np.array([True] * len(data)) - self._compare_other(data, comparison_op, other) + with tm.assert_produces_warning( + FutureWarning, match="incompatible dtype", check_stacklevel=False + ): + self._compare_other(data, comparison_op, other) other = pd.Series([True] * len(data)) - self._compare_other(data, comparison_op, other) + with tm.assert_produces_warning( + FutureWarning, match="incompatible dtype", check_stacklevel=False + ): + self._compare_other(data, comparison_op, other) @pytest.mark.parametrize("other", [True, False, pd.NA]) def test_scalar(self, other, comparison_op, dtype): diff --git a/pandas/tests/copy_view/test_methods.py b/pandas/tests/copy_view/test_methods.py index 7b8bc35f016b1..a04097409fe16 100644 --- a/pandas/tests/copy_view/test_methods.py +++ b/pandas/tests/copy_view/test_methods.py @@ -1370,15 +1370,18 @@ def test_putmask_aligns_rhs_no_reference(using_copy_on_write, dtype): assert np.shares_memory(arr_a, get_array(df, "a")) -@pytest.mark.parametrize("val, exp", [(5.5, True), (5, False)]) -def test_putmask_dont_copy_some_blocks(using_copy_on_write, val, exp): +@pytest.mark.parametrize( + "val, exp, warn", [(5.5, True, FutureWarning), (5, False, None)] +) +def test_putmask_dont_copy_some_blocks(using_copy_on_write, val, exp, warn): df = DataFrame({"a": [1, 2], "b": 1, "c": 1.5}) view = df[:] df_orig = df.copy() indexer = DataFrame( [[True, False, False], [True, False, False]], columns=list("abc") ) - df[indexer] = val + with tm.assert_produces_warning(warn, match="incompatible dtype"): + df[indexer] = val if using_copy_on_write: assert not np.shares_memory(get_array(view, "a"), get_array(df, "a")) diff --git a/pandas/tests/frame/indexing/test_setitem.py b/pandas/tests/frame/indexing/test_setitem.py index 4c8b93814e277..2f8956e1fddb5 100644 --- a/pandas/tests/frame/indexing/test_setitem.py +++ b/pandas/tests/frame/indexing/test_setitem.py @@ -988,7 +988,8 @@ def inc(x): return x + 1 df = DataFrame([[-1, 1], [1, -1]]) - df[df > 0] = inc + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df[df > 0] = inc expected = DataFrame([[-1, inc], [inc, -1]]) tm.assert_frame_equal(df, expected) diff --git a/pandas/tests/frame/methods/test_map.py b/pandas/tests/frame/methods/test_map.py index 596ef453b2e02..6be398fa13799 100644 --- a/pandas/tests/frame/methods/test_map.py +++ b/pandas/tests/frame/methods/test_map.py @@ -112,7 +112,8 @@ def test_map_na_ignore(float_frame): lambda x: len(str(x)), na_action="ignore" ) strlen_frame_with_na = strlen_frame.copy() - strlen_frame_with_na[mask] = pd.NA + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + strlen_frame_with_na[mask] = pd.NA tm.assert_frame_equal(strlen_frame_na_ignore, strlen_frame_with_na) diff --git a/pandas/tests/groupby/test_counting.py b/pandas/tests/groupby/test_counting.py index 921b42efb2bb3..e91e6d6336d75 100644 --- a/pandas/tests/groupby/test_counting.py +++ b/pandas/tests/groupby/test_counting.py @@ -334,7 +334,8 @@ def test_count_cross_type(): ) df = DataFrame(vals, columns=["a", "b", "c", "d"]) - df[df == 2] = np.nan + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df[df == 2] = np.nan expected = df.groupby(["c", "d"]).count() for t in ["float32", "object"]: diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index b347691c3c101..58a410a546068 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -277,7 +277,8 @@ def test_underlying_data_conversion(using_copy_on_write): df["val"] = 0 df_original = df.copy() df - df["val"].update(s) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df["val"].update(s) if using_copy_on_write: expected = df_original From b8532ccaa0da56346a280af9f0cf0c0f11d7e0ec Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 26 Apr 2023 11:15:46 +0100 Subject: [PATCH 48/80] one more --- pandas/tests/frame/indexing/test_mask.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/frame/indexing/test_mask.py b/pandas/tests/frame/indexing/test_mask.py index 233e2dcce81a7..7b4b7b469565e 100644 --- a/pandas/tests/frame/indexing/test_mask.py +++ b/pandas/tests/frame/indexing/test_mask.py @@ -147,6 +147,7 @@ def test_mask_inplace_no_other(): # GH#51685 df = DataFrame({"a": [1, 2], "b": ["x", "y"]}) cond = DataFrame({"a": [True, False], "b": [False, True]}) - df.mask(cond, inplace=True) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + df.mask(cond, inplace=True) expected = DataFrame({"a": [np.nan, 2], "b": ["x", np.nan]}) tm.assert_frame_equal(df, expected) From 81bba3c93edb0d91281bfb33e93a15a92f4f7aec Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 26 Apr 2023 11:19:01 +0100 Subject: [PATCH 49/80] simplify --- pandas/tests/series/indexing/test_setitem.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 1b49b29c69f69..0a4885c2642fb 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -1635,11 +1635,7 @@ def test_32878_complex_itemsize(): # GH#32878 used to coerce val to inf+0.000000e+00j # 'overflow encountered in cast' thrown by numpy - with tm.assert_produces_warning( - (FutureWarning, RuntimeWarning), - match="incompatible dtype|overflow encountered in cast", - check_stacklevel=False, - ): + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): ser[0] = val assert ser[0] == val expected = Series([val, 1, 2, 3, 4], dtype="c16") From 87922b536ff576117720a2ff5d5dba37e2b79dc9 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 26 Apr 2023 14:22:22 +0100 Subject: [PATCH 50/80] simplify and remove some false-positives --- pandas/core/internals/blocks.py | 34 ++++++++----------- pandas/tests/frame/indexing/test_where.py | 6 ++-- pandas/tests/series/indexing/test_indexing.py | 3 +- pandas/tests/series/indexing/test_setitem.py | 9 ++--- pandas/tests/series/methods/test_update.py | 11 +++--- 5 files changed, 25 insertions(+), 38 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index b403135544915..5c3d4affe8589 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -414,7 +414,7 @@ def split_and_operate(self, func, *args, **kwargs) -> list[Block]: # Up/Down-casting @final - def coerce_to_target_dtype(self, other) -> Block: + def coerce_to_target_dtype(self, other, warn_on_upcast: bool = False) -> Block: """ coerce the current block to a dtype compat for other we will return a block, possibly object, and not raise @@ -423,7 +423,15 @@ def coerce_to_target_dtype(self, other) -> Block: and will receive the same block """ new_dtype = find_result_type(self.values, other) - + if warn_on_upcast and self.dtype != new_dtype: + warnings.warn( + f"Setting an item of incompatible dtype is deprecated " + "and will raise in a future error of pandas. " + f"Value '{other}' has dtype incompatible with {self.dtype}, " + "please explicitly cast to a compatible dtype first.", + FutureWarning, + stacklevel=find_stack_level(), + ) return self.astype(new_dtype, copy=False) @final @@ -1049,15 +1057,7 @@ def setitem(self, indexer, value, using_cow: bool = False) -> Block: casted = np_can_hold_element(values.dtype, value) except LossySetitemError: # current dtype cannot store value, coerce to common dtype - warnings.warn( - f"Setting an item of incompatible dtype is deprecated " - "and will raise in a future error of pandas. " - f"Value '{value}' has dtype incompatible with {values.dtype}, " - "please explicitly cast to a compatible dtype first.", - FutureWarning, - stacklevel=find_stack_level(), - ) - nb = self.coerce_to_target_dtype(value) + nb = self.coerce_to_target_dtype(value, warn_on_upcast=True) return nb.setitem(indexer, value) else: if self.dtype == _dtype_obj: @@ -1123,20 +1123,14 @@ def putmask(self, mask, new, using_cow: bool = False) -> list[Block]: return [self.copy(deep=False)] return [self] except LossySetitemError: - warnings.warn( - f"Setting an item of incompatible dtype is deprecated " - "and will raise in a future error of pandas. " - f"Value '{new}' has dtype incompatible with {values.dtype}, " - "please explicitly cast to a compatible dtype first.", - FutureWarning, - stacklevel=find_stack_level(), - ) if self.ndim == 1 or self.shape[0] == 1: # no need to split columns if not is_list_like(new): # using just new[indexer] can't save us the need to cast - return self.coerce_to_target_dtype(new).putmask(mask, new) + return self.coerce_to_target_dtype( + new, warn_on_upcast=True + ).putmask(mask, new) else: indexer = mask.nonzero()[0] nb = self.setitem(indexer, new[indexer], using_cow=using_cow) diff --git a/pandas/tests/frame/indexing/test_where.py b/pandas/tests/frame/indexing/test_where.py index a87a37a5a5bf5..8603ac1eae206 100644 --- a/pandas/tests/frame/indexing/test_where.py +++ b/pandas/tests/frame/indexing/test_where.py @@ -356,8 +356,7 @@ def test_where_bug_transposition(self): do_not_replace = b.isna() | (a > b) expected = a.copy() - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - expected[~do_not_replace] = b + expected[~do_not_replace] = b result = a.where(do_not_replace, b) tm.assert_frame_equal(result, expected) @@ -367,8 +366,7 @@ def test_where_bug_transposition(self): do_not_replace = b.isna() | (a > b) expected = a.copy() - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - expected[~do_not_replace] = b + expected[~do_not_replace] = b result = a.where(do_not_replace, b) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 58a410a546068..b347691c3c101 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -277,8 +277,7 @@ def test_underlying_data_conversion(using_copy_on_write): df["val"] = 0 df_original = df.copy() df - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - df["val"].update(s) + df["val"].update(s) if using_copy_on_write: expected = df_original diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 0a4885c2642fb..5f5dd23714cb7 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -251,8 +251,7 @@ def test_setitem_mask_cast(self): # GH#2746 # need to upcast ser = Series([1, 2], index=[1, 2], dtype="int64") - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - ser[[True, False]] = Series([0], index=[1], dtype="int64") + ser[[True, False]] = Series([0], index=[1], dtype="int64") expected = Series([0, 2], index=[1, 2], dtype="int64") tm.assert_series_equal(ser, expected) @@ -410,14 +409,12 @@ def test_setitem_mask_smallint_no_upcast(self): mask = np.array([True, False, True]) ser = orig.copy() - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - ser[mask] = alt + ser[mask] = alt expected = Series([245, 2, 246], dtype="uint8") tm.assert_series_equal(ser, expected) ser2 = orig.copy() - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - ser2.mask(mask, alt, inplace=True) + ser2.mask(mask, alt, inplace=True) tm.assert_series_equal(ser2, expected) # FIXME: don't leave commented-out diff --git a/pandas/tests/series/methods/test_update.py b/pandas/tests/series/methods/test_update.py index 3ae8f233625a3..b06ed153bfe49 100644 --- a/pandas/tests/series/methods/test_update.py +++ b/pandas/tests/series/methods/test_update.py @@ -41,13 +41,13 @@ def test_update(self, using_copy_on_write): "other, dtype, expected, warn", [ # other is int - ([61, 63], "int32", Series([10, 61, 12], dtype="int32"), FutureWarning), - ([61, 63], "int64", Series([10, 61, 12]), FutureWarning), + ([61, 63], "int32", Series([10, 61, 12], dtype="int32"), None), + ([61, 63], "int64", Series([10, 61, 12]), None), ([61, 63], float, Series([10.0, 61.0, 12.0]), None), ([61, 63], object, Series([10, 61, 12], dtype=object), None), # other is float, but can be cast to int - ([61.0, 63.0], "int32", Series([10, 61, 12], dtype="int32"), FutureWarning), - ([61.0, 63.0], "int64", Series([10, 61, 12]), FutureWarning), + ([61.0, 63.0], "int32", Series([10, 61, 12], dtype="int32"), None), + ([61.0, 63.0], "int64", Series([10, 61, 12]), None), ([61.0, 63.0], float, Series([10.0, 61.0, 12.0]), None), ([61.0, 63.0], object, Series([10, 61.0, 12], dtype=object), None), # others is float, cannot be cast to int @@ -85,8 +85,7 @@ def test_update_dtypes(self, other, dtype, expected, warn): ) def test_update_from_non_series(self, series, other, expected): # GH 33215 - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - series.update(other) + series.update(other) tm.assert_series_equal(series, expected) @pytest.mark.parametrize( From f314dd1d96cc367feecb017a662fe33098d099e6 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 27 Apr 2023 10:08:32 +0100 Subject: [PATCH 51/80] clean up --- pandas/_libs/tslibs/timedeltas.pyx | 3 ++- pandas/tests/series/indexing/test_setitem.py | 1 - pandas/tests/series/methods/test_convert_dtypes.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 563c2505a1812..621bc5b7daf19 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -34,6 +34,7 @@ import_datetime() cimport pandas._libs.tslibs.util as util +from pandas._libs.missing cimport checknull_with_nat_and_na from pandas._libs.tslibs.base cimport ABCTimestamp from pandas._libs.tslibs.conversion cimport ( cast_from_unit, @@ -342,7 +343,7 @@ cdef convert_to_timedelta64(object ts, str unit): """ # Caller is responsible for checking unit not in ["Y", "y", "M"] - if checknull_with_nat(ts): + if checknull_with_nat_and_na(ts): return np.timedelta64(NPY_NAT, "ns") elif isinstance(ts, _Timedelta): # already in the proper format diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 5f5dd23714cb7..a984f02beb9cf 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -1631,7 +1631,6 @@ def test_32878_complex_itemsize(): val = val.astype("c16") # GH#32878 used to coerce val to inf+0.000000e+00j - # 'overflow encountered in cast' thrown by numpy with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): ser[0] = val assert ser[0] == val diff --git a/pandas/tests/series/methods/test_convert_dtypes.py b/pandas/tests/series/methods/test_convert_dtypes.py index f4523d19bdf16..fcc298d063569 100644 --- a/pandas/tests/series/methods/test_convert_dtypes.py +++ b/pandas/tests/series/methods/test_convert_dtypes.py @@ -206,8 +206,7 @@ def test_convert_dtypes( with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): result[result.notna()] = np.nan else: - with tm.assert_produces_warning(None): - result[result.notna()] = np.nan + result[result.notna()] = np.nan # Make sure original not changed tm.assert_series_equal(series, copy) From 8cad20150827ec1737a688400773701a785ac56d Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 27 Apr 2023 10:17:23 +0100 Subject: [PATCH 52/80] remove final filterwarnings --- pandas/tests/indexing/test_indexing.py | 27 +++++++++++++++----- pandas/tests/interchange/test_impl.py | 7 ++--- pandas/tests/series/indexing/test_setitem.py | 2 -- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 18c33388bde1a..eda22a29859c9 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -811,34 +811,49 @@ class TestDataframeNoneCoercion: ] @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) - @pytest.mark.filterwarnings("ignore:.*item of incompatible dtype.*:FutureWarning") def test_coercion_with_loc(self, expected): start_data, expected_result = expected start_dataframe = DataFrame({"foo": start_data}) - start_dataframe.loc[0, ["foo"]] = None + if start_dataframe["foo"].dtype == "int": + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + start_dataframe.loc[0, ["foo"]] = None + else: + start_dataframe.loc[0, ["foo"]] = None expected_dataframe = DataFrame({"foo": expected_result}) tm.assert_frame_equal(start_dataframe, expected_dataframe) @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) - @pytest.mark.filterwarnings("ignore:.*item of incompatible dtype.*:FutureWarning") def test_coercion_with_setitem_and_dataframe(self, expected): start_data, expected_result = expected start_dataframe = DataFrame({"foo": start_data}) - start_dataframe[start_dataframe["foo"] == start_dataframe["foo"][0]] = None + if start_dataframe["foo"].dtype == "int": + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + start_dataframe[ + start_dataframe["foo"] == start_dataframe["foo"][0] + ] = None + else: + start_dataframe[start_dataframe["foo"] == start_dataframe["foo"][0]] = None expected_dataframe = DataFrame({"foo": expected_result}) tm.assert_frame_equal(start_dataframe, expected_dataframe) @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) - @pytest.mark.filterwarnings("ignore:.*item of incompatible dtype.*:FutureWarning") def test_none_coercion_loc_and_dataframe(self, expected): start_data, expected_result = expected start_dataframe = DataFrame({"foo": start_data}) - start_dataframe.loc[start_dataframe["foo"] == start_dataframe["foo"][0]] = None + if start_dataframe["foo"].dtype == "int": + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + start_dataframe.loc[ + start_dataframe["foo"] == start_dataframe["foo"][0] + ] = None + else: + start_dataframe.loc[ + start_dataframe["foo"] == start_dataframe["foo"][0] + ] = None expected_dataframe = DataFrame({"foo": expected_result}) tm.assert_frame_equal(start_dataframe, expected_dataframe) diff --git a/pandas/tests/interchange/test_impl.py b/pandas/tests/interchange/test_impl.py index a3fd3e91477dd..5e526cd23f798 100644 --- a/pandas/tests/interchange/test_impl.py +++ b/pandas/tests/interchange/test_impl.py @@ -174,10 +174,11 @@ def test_missing_from_masked(): rng.choice(np.arange(len(df)), size=num_nulls, replace=False) ] if is_float_dtype(df[col]): - warn = None + with tm.assert_produces_warning( + FutureWarning, match="item of incompatible dtype" + ): + df.loc[null_idx, col] = None else: - warn = FutureWarning - with tm.assert_produces_warning(warn, match="item of incompatible dtype"): df.loc[null_idx, col] = None df2 = df.__dataframe__() diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index a984f02beb9cf..a768f24b95df3 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -786,8 +786,6 @@ def test_int_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): def test_slice_key(self, obj, key, expected, warn, val, indexer_sli, is_inplace): if not isinstance(key, slice): return - # if isinstance(key, slice) and key.start is None and key.stop is None: - # warn = None if indexer_sli is not tm.loc: # Note: no .loc because that handles slice edges differently From d3b12abf72e7afd9331bfdee4ddd78a5c669c638 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 27 Apr 2023 10:19:16 +0100 Subject: [PATCH 53/80] undo unrelated change --- pandas/_libs/tslibs/timedeltas.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 621bc5b7daf19..563c2505a1812 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -34,7 +34,6 @@ import_datetime() cimport pandas._libs.tslibs.util as util -from pandas._libs.missing cimport checknull_with_nat_and_na from pandas._libs.tslibs.base cimport ABCTimestamp from pandas._libs.tslibs.conversion cimport ( cast_from_unit, @@ -343,7 +342,7 @@ cdef convert_to_timedelta64(object ts, str unit): """ # Caller is responsible for checking unit not in ["Y", "y", "M"] - if checknull_with_nat_and_na(ts): + if checknull_with_nat(ts): return np.timedelta64(NPY_NAT, "ns") elif isinstance(ts, _Timedelta): # already in the proper format From adc00225b439dc04c6d3e1a043a06c6d735fe87b Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 27 Apr 2023 10:20:25 +0100 Subject: [PATCH 54/80] fixup raises_chained_assignment_error --- pandas/_testing/contexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_testing/contexts.py b/pandas/_testing/contexts.py index 7fabd467295f6..7e443e08c80ba 100644 --- a/pandas/_testing/contexts.py +++ b/pandas/_testing/contexts.py @@ -206,7 +206,7 @@ def use_numexpr(use, min_elements=None) -> Generator[None, None, None]: def raises_chained_assignment_error(extra_warnings=()): - if PYPY and extra_warnings is None: + if PYPY and not extra_warnings: from contextlib import nullcontext return nullcontext() From d5bdfcfc5d5bee0eed9db9c9c295aedf842a32f5 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 27 Apr 2023 10:30:40 +0100 Subject: [PATCH 55/80] remove another filterwarnings --- pandas/tests/indexing/test_indexing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index eda22a29859c9..edde22a3d0170 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -661,7 +661,6 @@ def test_loc_setitem_fullindex_views(self): df.loc[df.index] = df.loc[df.index] tm.assert_frame_equal(df, df2) - @pytest.mark.filterwarnings("ignore:.*item of incompatible dtype.*:FutureWarning") def test_rhs_alignment(self): # GH8258, tests that both rows & columns are aligned to what is # assigned to. covers both uniform data-type & multi-type cases @@ -706,7 +705,8 @@ def run_tests(df, rhs, right_loc, right_iloc): frame["jolie"] = frame["jolie"].map(lambda x: f"@{x}") right_iloc["joe"] = [1.0, "@-28", "@-20", "@-12", 17.0] right_iloc["jolie"] = ["@2", -26.0, -18.0, -10.0, "@18"] - run_tests(df, rhs, right_loc, right_iloc) + with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): + run_tests(df, rhs, right_loc, right_iloc) @pytest.mark.parametrize( "idx", [_mklbl("A", 20), np.arange(20) + 100, np.linspace(100, 150, 20)] From dfa8ed26702e3761fc88793cd91bfc5bcf46baba Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 27 Apr 2023 15:10:25 +0100 Subject: [PATCH 56/80] fixup interchange test --- pandas/tests/interchange/test_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/interchange/test_impl.py b/pandas/tests/interchange/test_impl.py index 5e526cd23f798..9b2a547288335 100644 --- a/pandas/tests/interchange/test_impl.py +++ b/pandas/tests/interchange/test_impl.py @@ -173,7 +173,7 @@ def test_missing_from_masked(): null_idx = df.index[ rng.choice(np.arange(len(df)), size=num_nulls, replace=False) ] - if is_float_dtype(df[col]): + if not is_float_dtype(df[col]): with tm.assert_produces_warning( FutureWarning, match="item of incompatible dtype" ): From 3efe0a5cc25a03c999e89a050d2c92f38a7b2eeb Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 27 Apr 2023 18:44:39 +0100 Subject: [PATCH 57/80] better parametrisation --- pandas/tests/indexing/test_indexing.py | 32 ++++++++------------------ 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index edde22a3d0170..3a090f86b6c57 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -799,26 +799,24 @@ def test_label_indexing_on_nan(self, nulls_fixture): class TestDataframeNoneCoercion: EXPECTED_SINGLE_ROW_RESULTS = [ # For numeric series, we should coerce to NaN. - ([1, 2, 3], [np.nan, 2, 3]), - ([1.0, 2.0, 3.0], [np.nan, 2.0, 3.0]), + ([1, 2, 3], [np.nan, 2, 3], FutureWarning), + ([1.0, 2.0, 3.0], [np.nan, 2.0, 3.0], None), # For datetime series, we should coerce to NaT. ( [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)], [NaT, datetime(2000, 1, 2), datetime(2000, 1, 3)], + None, ), # For objects, we should preserve the None value. - (["foo", "bar", "baz"], [None, "bar", "baz"]), + (["foo", "bar", "baz"], [None, "bar", "baz"], None), ] @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) def test_coercion_with_loc(self, expected): - start_data, expected_result = expected + start_data, expected_result, warn = expected start_dataframe = DataFrame({"foo": start_data}) - if start_dataframe["foo"].dtype == "int": - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - start_dataframe.loc[0, ["foo"]] = None - else: + with tm.assert_produces_warning(warn, match="incompatible dtype"): start_dataframe.loc[0, ["foo"]] = None expected_dataframe = DataFrame({"foo": expected_result}) @@ -826,15 +824,10 @@ def test_coercion_with_loc(self, expected): @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) def test_coercion_with_setitem_and_dataframe(self, expected): - start_data, expected_result = expected + start_data, expected_result, warn = expected start_dataframe = DataFrame({"foo": start_data}) - if start_dataframe["foo"].dtype == "int": - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - start_dataframe[ - start_dataframe["foo"] == start_dataframe["foo"][0] - ] = None - else: + with tm.assert_produces_warning(warn, match="incompatible dtype"): start_dataframe[start_dataframe["foo"] == start_dataframe["foo"][0]] = None expected_dataframe = DataFrame({"foo": expected_result}) @@ -842,15 +835,10 @@ def test_coercion_with_setitem_and_dataframe(self, expected): @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS) def test_none_coercion_loc_and_dataframe(self, expected): - start_data, expected_result = expected + start_data, expected_result, warn = expected start_dataframe = DataFrame({"foo": start_data}) - if start_dataframe["foo"].dtype == "int": - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - start_dataframe.loc[ - start_dataframe["foo"] == start_dataframe["foo"][0] - ] = None - else: + with tm.assert_produces_warning(warn, match="incompatible dtype"): start_dataframe.loc[ start_dataframe["foo"] == start_dataframe["foo"][0] ] = None From 4ba259dd13113ff23362ee3fb23081f809a0940f Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 4 May 2023 10:36:26 +0100 Subject: [PATCH 58/80] okwarning => codeblock --- doc/source/whatsnew/v0.21.0.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v0.21.0.rst b/doc/source/whatsnew/v0.21.0.rst index 4048dd248830e..c74da79cb4919 100644 --- a/doc/source/whatsnew/v0.21.0.rst +++ b/doc/source/whatsnew/v0.21.0.rst @@ -425,13 +425,12 @@ Note that this also changes the sum of an empty ``Series``. Previously this alwa In [1]: pd.Series([]).sum() Out[1]: 0 -but for consistency with the all-NaN case, this was changed to return NaN as well: +but for consistency with the all-NaN case, this was changed to return 0 as well: -.. ipython:: python - :okwarning: - - pd.Series([]).sum() +.. code-block:: ipython + In [2]: pd.Series([]).sum() + Out[2]: 0 .. _whatsnew_0210.api_breaking.loc: From c21aa4dbac78533feda496c05583319e064efa31 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 4 May 2023 10:48:35 +0100 Subject: [PATCH 59/80] okwarning => codeblock in v1.3.0 --- doc/source/whatsnew/v1.3.0.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index ad6d3afa7cbb6..0099025343d5d 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -501,14 +501,17 @@ Consistent casting with setting into Boolean Series Setting non-boolean values into a :class:`Series` with ``dtype=bool`` now consistently casts to ``dtype=object`` (:issue:`38709`) -.. ipython:: python - :okwarning: +.. code-block:: ipython + + In [1]: orig = pd.Series([True, False]) + + In [2]: ser = orig.copy() + + In [3]: ser.iloc[1] = np.nan + + In [4]: ser2 = orig.copy() - orig = pd.Series([True, False]) - ser = orig.copy() - ser.iloc[1] = np.nan - ser2 = orig.copy() - ser2.iloc[1] = 2.0 + In [5]: ser2.iloc[1] = 2.0 *Previous behavior*: From 05ffc27bdc2f0266fcf856a8187f1ad39e990a23 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 4 May 2023 10:54:35 +0100 Subject: [PATCH 60/80] one more codeblock --- doc/source/whatsnew/v0.21.0.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v0.21.0.rst b/doc/source/whatsnew/v0.21.0.rst index c74da79cb4919..90fdd6720fb07 100644 --- a/doc/source/whatsnew/v0.21.0.rst +++ b/doc/source/whatsnew/v0.21.0.rst @@ -754,11 +754,16 @@ Previously assignments, ``.where()`` and ``.fillna()`` with a ``bool`` assignmen New behavior -.. ipython:: python - :okwarning: +.. codeblock:: ipython - s[1] = True - s + In [7]: s[1] = True + + In [8]: s + Out[8]: + 0 1 + 1 True + 2 3 + Length: 3, dtype: object Previously, as assignment to a datetimelike with a non-datetimelike would coerce the non-datetime-like item being assigned (:issue:`14145`). From 6612690dece41e12d1fadbdfb882f10a44cb2cf9 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 4 May 2023 11:00:48 +0100 Subject: [PATCH 61/80] avoid upcast --- pandas/tests/arrays/masked_shared.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/arrays/masked_shared.py b/pandas/tests/arrays/masked_shared.py index 831fc64512b98..a2bbe1d01c1a5 100644 --- a/pandas/tests/arrays/masked_shared.py +++ b/pandas/tests/arrays/masked_shared.py @@ -24,11 +24,12 @@ def _compare_other(self, data, op, other): ser = pd.Series(data) result = op(ser, other) - expected = op(pd.Series(data._data), other) + # Set nullable dtype here to avoid upcasting when setting to pd.NA below + expected = op(pd.Series(data._data), other).astype("boolean") # fill the nan locations expected[data._mask] = pd.NA - expected = expected.astype("boolean") + expected = expected tm.assert_series_equal(result, expected) From adaf4e410f2294da298b35e2daf3c4ac616ab1a6 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 4 May 2023 15:15:08 +0100 Subject: [PATCH 62/80] post-merge fixup --- pandas/tests/arrays/boolean/test_comparison.py | 15 +++------------ pandas/tests/frame/indexing/test_mask.py | 3 +-- pandas/tests/frame/methods/test_asof.py | 3 +-- pandas/tests/groupby/test_counting.py | 3 +-- 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/pandas/tests/arrays/boolean/test_comparison.py b/pandas/tests/arrays/boolean/test_comparison.py index d265c43aa20f8..2eeb9da574b1e 100644 --- a/pandas/tests/arrays/boolean/test_comparison.py +++ b/pandas/tests/arrays/boolean/test_comparison.py @@ -24,24 +24,15 @@ def dtype(): class TestComparisonOps(ComparisonOps): def test_compare_scalar(self, data, comparison_op): - with tm.assert_produces_warning( - FutureWarning, match="incompatible dtype", check_stacklevel=False - ): - self._compare_other(data, comparison_op, True) + self._compare_other(data, comparison_op, True) def test_compare_array(self, data, comparison_op): other = pd.array([True] * len(data), dtype="boolean") self._compare_other(data, comparison_op, other) other = np.array([True] * len(data)) - with tm.assert_produces_warning( - FutureWarning, match="incompatible dtype", check_stacklevel=False - ): - self._compare_other(data, comparison_op, other) + self._compare_other(data, comparison_op, other) other = pd.Series([True] * len(data)) - with tm.assert_produces_warning( - FutureWarning, match="incompatible dtype", check_stacklevel=False - ): - self._compare_other(data, comparison_op, other) + self._compare_other(data, comparison_op, other) @pytest.mark.parametrize("other", [True, False, pd.NA]) def test_scalar(self, other, comparison_op, dtype): diff --git a/pandas/tests/frame/indexing/test_mask.py b/pandas/tests/frame/indexing/test_mask.py index 01f4b79baff40..ed0bf256d1ee7 100644 --- a/pandas/tests/frame/indexing/test_mask.py +++ b/pandas/tests/frame/indexing/test_mask.py @@ -147,7 +147,6 @@ def test_mask_inplace_no_other(): # GH#51685 df = DataFrame({"a": [1.0, 2.0], "b": ["x", "y"]}) cond = DataFrame({"a": [True, False], "b": [False, True]}) - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - df.mask(cond, inplace=True) + df.mask(cond, inplace=True) expected = DataFrame({"a": [np.nan, 2], "b": ["x", np.nan]}) tm.assert_frame_equal(df, expected) diff --git a/pandas/tests/frame/methods/test_asof.py b/pandas/tests/frame/methods/test_asof.py index baadfdc8f085b..09cb3fbe1bacb 100644 --- a/pandas/tests/frame/methods/test_asof.py +++ b/pandas/tests/frame/methods/test_asof.py @@ -89,8 +89,7 @@ def test_missing(self, date_range_frame): ) tm.assert_series_equal(result, expected) - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - result = df.asof(to_datetime(["1989-12-31"])) + result = df.asof(to_datetime(["1989-12-31"])) expected = DataFrame( index=to_datetime(["1989-12-31"]), columns=["A", "B"], dtype="float64" ) diff --git a/pandas/tests/groupby/test_counting.py b/pandas/tests/groupby/test_counting.py index 96c9f2cfb5596..97e4e8a852429 100644 --- a/pandas/tests/groupby/test_counting.py +++ b/pandas/tests/groupby/test_counting.py @@ -333,8 +333,7 @@ def test_count_cross_type(): ).astype("float64") df = DataFrame(vals, columns=["a", "b", "c", "d"]) - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - df[df == 2] = np.nan + df[df == 2] = np.nan expected = df.groupby(["c", "d"]).count() for t in ["float32", "object"]: From f194434e872394522c77267ed7321d7826c52a28 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 4 May 2023 15:18:49 +0100 Subject: [PATCH 63/80] docs fixup; --- doc/source/whatsnew/v1.3.0.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 0099025343d5d..17aab87b93f8e 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -531,10 +531,19 @@ casts to ``dtype=object`` (:issue:`38709`) *New behavior*: -.. ipython:: python +.. code-block:: ipython + + In [1]: ser + Out [1]: + 0 True + 1 NaN + dtype: object - ser - ser2 + In [2]:ser2 + Out [2]: + 0 True + 1 2.0 + dtype: object .. _whatsnew_130.notable_bug_fixes.rolling_groupby_column: From 82ecca2449017396feec20034eba8ffddfa40f6f Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 5 May 2023 13:04:50 +0100 Subject: [PATCH 64/80] post-merge fixup --- pandas/tests/indexing/test_at.py | 5 +---- pandas/tests/series/methods/test_update.py | 3 +-- pandas/tests/series/test_missing.py | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pandas/tests/indexing/test_at.py b/pandas/tests/indexing/test_at.py index 06c424c9447f4..01315647c464b 100644 --- a/pandas/tests/indexing/test_at.py +++ b/pandas/tests/indexing/test_at.py @@ -123,10 +123,7 @@ def test_at_datetime_index(self, row): index=DatetimeIndex(data=["2019-01-01", "2019-01-02"]), ) - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - df.at[row, 0] = 0.5 + df.at[row, 0] = 0.5 tm.assert_frame_equal(df, expected) diff --git a/pandas/tests/series/methods/test_update.py b/pandas/tests/series/methods/test_update.py index f2ff7120358ae..c91bdbc6bcee2 100644 --- a/pandas/tests/series/methods/test_update.py +++ b/pandas/tests/series/methods/test_update.py @@ -29,8 +29,7 @@ def test_update(self, using_copy_on_write): df["c"] = df["c"].astype(object) df_orig = df.copy() - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - df["c"].update(Series(["foo"], index=[0])) + df["c"].update(Series(["foo"], index=[0])) if using_copy_on_write: expected = df_orig else: diff --git a/pandas/tests/series/test_missing.py b/pandas/tests/series/test_missing.py index 904745cd30db4..6cf67e22b6f19 100644 --- a/pandas/tests/series/test_missing.py +++ b/pandas/tests/series/test_missing.py @@ -98,6 +98,5 @@ def test_hasnans_uncached_for_series(): ser = idx.to_series() assert ser.hasnans is False assert not hasattr(ser, "_cache") - with tm.assert_produces_warning(FutureWarning, match="item of incompatible dtype"): - ser.iloc[-1] = np.nan + ser.iloc[-1] = np.nan assert ser.hasnans is True From a9d58911cce29ecbd07c089ca9b7a7fc2df8b8b6 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Fri, 5 May 2023 13:33:23 +0100 Subject: [PATCH 65/80] remove more upcasts --- pandas/tests/frame/indexing/test_where.py | 33 +++++++------------- pandas/tests/indexing/test_indexing.py | 17 +++------- pandas/tests/indexing/test_loc.py | 19 +++-------- pandas/tests/interchange/test_impl.py | 12 ++----- pandas/tests/series/indexing/test_setitem.py | 12 +++---- 5 files changed, 27 insertions(+), 66 deletions(-) diff --git a/pandas/tests/frame/indexing/test_where.py b/pandas/tests/frame/indexing/test_where.py index 8603ac1eae206..1a28ddfbb3faf 100644 --- a/pandas/tests/frame/indexing/test_where.py +++ b/pandas/tests/frame/indexing/test_where.py @@ -164,7 +164,7 @@ def test_where_invalid(self): with pytest.raises(ValueError, match=msg): df.mask(0) - def test_where_set(self, where_frame, float_string_frame): + def test_where_set(self, where_frame, float_string_frame, mixed_int_frame): # where inplace def _check_set(df, cond, check_dtypes=True): @@ -189,24 +189,18 @@ def _check_set(df, cond, check_dtypes=True): with pytest.raises(TypeError, match=msg): df > 0 return + if df is mixed_int_frame: + df = df.astype("float64") cond = df > 0 - if (df.dtypes == "int").any(): - warn = FutureWarning - else: - warn = None - - with tm.assert_produces_warning(warn, match="incompatible dtype"): - _check_set(df, cond) + _check_set(df, cond) cond = df >= 0 - with tm.assert_produces_warning(warn, match="incompatible dtype"): - _check_set(df, cond) + _check_set(df, cond) # aligning cond = (df >= 0)[1:] - with tm.assert_produces_warning(warn, match="incompatible dtype"): - _check_set(df, cond) + _check_set(df, cond) def test_where_series_slicing(self): # GH 10218 @@ -336,16 +330,14 @@ def test_where_bug_mixed(self, any_signed_int_numpy_dtype): ) expected = DataFrame( - {"a": [np.nan, np.nan, 3.0, 4.0], "b": [4.0, 3.0, np.nan, np.nan]}, - dtype="float64", - ) + {"a": [-1, -1, 3, 4], "b": [4.0, 3.0, -1, -1]}, + ).astype({"a": any_signed_int_numpy_dtype, "b": "float64"}) - result = df.where(df > 2, np.nan) + result = df.where(df > 2, -1) tm.assert_frame_equal(result, expected) result = df.copy() - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - return_value = result.where(result > 2, np.nan, inplace=True) + return_value = result.where(result > 2, -1, inplace=True) assert return_value is None tm.assert_frame_equal(result, expected) @@ -1041,9 +1033,8 @@ def test_where_int_overflow(replacement): def test_where_inplace_no_other(): # GH#51685 - df = DataFrame({"a": [1, 2], "b": ["x", "y"]}) + df = DataFrame({"a": [1.0, 2.0], "b": ["x", "y"]}) cond = DataFrame({"a": [True, False], "b": [False, True]}) - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - df.where(cond, inplace=True) + df.where(cond, inplace=True) expected = DataFrame({"a": [1, np.nan], "b": [np.nan, "y"]}) tm.assert_frame_equal(df, expected) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 3a090f86b6c57..f7977a6430fe9 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -445,7 +445,7 @@ def test_multi_assign(self): "col1": list(range(6)), "col2": list(range(6, 12)), } - ) + ).astype({"col2": "float64"}) df.iloc[1, 0] = np.nan df2 = df.copy() @@ -453,10 +453,7 @@ def test_multi_assign(self): cols = ["col1", "col2"] dft = df2 * 2 - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - dft.iloc[3, 3] = np.nan + dft.iloc[3, 3] = np.nan expected = DataFrame( { @@ -468,10 +465,7 @@ def test_multi_assign(self): ) # frame on rhs - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - df2.loc[mask, cols] = dft.loc[mask, cols] + df2.loc[mask, cols] = dft.loc[mask, cols] tm.assert_frame_equal(df2, expected) # with an ndarray on rhs @@ -486,10 +480,7 @@ def test_multi_assign(self): } ) df2 = df.copy() - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - df2.loc[mask, cols] = dft.loc[mask, cols].values + df2.loc[mask, cols] = dft.loc[mask, cols].values tm.assert_frame_equal(df2, expected) def test_multi_assign_broadcasting_rhs(self): diff --git a/pandas/tests/indexing/test_loc.py b/pandas/tests/indexing/test_loc.py index aee7a6df6d4c9..db9ad0fe7db0a 100644 --- a/pandas/tests/indexing/test_loc.py +++ b/pandas/tests/indexing/test_loc.py @@ -866,11 +866,8 @@ def test_loc_setitem_with_scalar_index(self, indexer, value): # assigning like "df.loc[0, ['A']] = ['Z']" should be evaluated # elementwisely, not using "setter('A', ['Z'])". - df = DataFrame([[1, 2], [3, 4]], columns=["A", "B"]) - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - df.loc[0, indexer] = value + df = DataFrame([[1, 2], [3, 4]], columns=["A", "B"]).astype({"A": object}) + df.loc[0, indexer] = value result = df.loc[0, "A"] assert is_scalar(result) and result == "Z" @@ -1533,7 +1530,7 @@ def test_loc_setitem_td64_non_nano(self): def test_loc_setitem_2d_to_1d_raises(self): data = np.random.randn(2, 2) - ser = Series(range(2)) + ser = Series(range(2), dtype="float64") msg = "|".join( [ @@ -1542,17 +1539,11 @@ def test_loc_setitem_2d_to_1d_raises(self): ] ) with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - ser.loc[range(2)] = data + ser.loc[range(2)] = data msg = r"could not broadcast input array from shape \(2,2\) into shape \(2,?\)" with pytest.raises(ValueError, match=msg): - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - ser.loc[:] = data + ser.loc[:] = data def test_loc_getitem_interval_index(self): # GH#19977 diff --git a/pandas/tests/interchange/test_impl.py b/pandas/tests/interchange/test_impl.py index c1bf684235340..6c9301264d562 100644 --- a/pandas/tests/interchange/test_impl.py +++ b/pandas/tests/interchange/test_impl.py @@ -7,8 +7,6 @@ from pandas._libs.tslibs import iNaT import pandas.util._test_decorators as td -from pandas.core.dtypes.common import is_float_dtype - import pandas as pd import pandas._testing as tm from pandas.core.interchange.column import PandasColumn @@ -173,7 +171,7 @@ def test_missing_from_masked(): { "x": np.array([1.0, 2.0, 3.0, 4.0, 0.0]), "y": np.array([1.5, 2.5, 3.5, 4.5, 0]), - "z": np.array([True, False, True, True, True]), + "z": np.array([True, False, True, True, True]).astype("float64"), } ) @@ -185,13 +183,7 @@ def test_missing_from_masked(): null_idx = df.index[ rng.choice(np.arange(len(df)), size=num_nulls, replace=False) ] - if not is_float_dtype(df[col]): - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - df.loc[null_idx, col] = None - else: - df.loc[null_idx, col] = None + df.loc[null_idx, col] = None df2 = df.__dataframe__() diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index a768f24b95df3..3b43de073e399 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -61,11 +61,8 @@ def test_setitem_multiindex_empty_slice(self): def test_setitem_with_string_index(self): # GH#23451 - ser = Series([1, 2, 3], index=["Date", "b", "other"]) - with tm.assert_produces_warning( - FutureWarning, match="item of incompatible dtype" - ): - ser["Date"] = date.today() + ser = Series([1, 2, 3], index=["Date", "b", "other"], dtype=object) + ser["Date"] = date.today() assert ser.Date == date.today() assert ser["Date"] == date.today() @@ -480,9 +477,8 @@ def test_setitem_callable_other(self): # GH#13299 inc = lambda x: x + 1 - ser = Series([1, 2, -1, 4]) - with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"): - ser[ser < 0] = inc + ser = Series([1, 2, -1, 4], dtype=object) + ser[ser < 0] = inc expected = Series([1, 2, inc, 4]) tm.assert_series_equal(ser, expected) From aec0c87e68858fb100ba71eb109ea829f9a5eeab Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 8 May 2023 15:05:42 +0100 Subject: [PATCH 66/80] adapt test from EA types --- pandas/tests/indexing/test_indexing.py | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 28f9b1a62172e..5c5f13e417842 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1110,3 +1110,52 @@ def test_scalar_setitem_series_with_nested_value_length1(value, indexer_sli): assert (ser.loc[0] == value).all() else: assert ser.loc[0] == value + + +class TestSetitemValidation: + # This is adapted from pandas/tests/arrays/masked/test_indexing.py + def _check_setitem_invalid(self, arr, invalid): + msg = "Setting an item of incompatible dtype is deprecated" + msg = re.escape(msg) + with tm.assert_produces_warning(FutureWarning, match=msg): + arr[0] = invalid + + # FIXME: don't leave commented-out + # with tm.assert_produces_warning(FutureWarning, match=msg): + # arr[:] = invalid + + # with tm.assert_produces_warning(FutureWarning, match=msg): + # arr[[0]] = invalid + + # with pytest.raises(TypeError): + # arr[[0]] = [invalid] + + # with pytest.raises(TypeError): + # arr[[0]] = np.array([invalid], dtype=object) + + _invalid_scalars = [ + 1 + 2j, + "True", + "1", + "1.0", + NaT, + np.datetime64("NaT"), + np.timedelta64("NaT"), + ] + + @pytest.mark.parametrize( + "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] + ) + def test_setitem_validation_scalar_bool(self, invalid): + arr = Series([True, False, None], dtype="bool") + self._check_setitem_invalid(arr, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) + def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): + arr = Series([1, 2, 3], dtype=any_int_numpy_dtype) + self._check_setitem_invalid(arr, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + arr = Series([1, 2, None], dtype=float_numpy_dtype) + self._check_setitem_invalid(arr, invalid) From 72e5609429f97220426ae1ecbdd73f2ecb5716bc Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 8 May 2023 15:07:45 +0100 Subject: [PATCH 67/80] move test to series/indexing --- pandas/tests/indexing/test_indexing.py | 49 ------------------ pandas/tests/series/indexing/test_indexing.py | 50 +++++++++++++++++++ 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 5c5f13e417842..28f9b1a62172e 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1110,52 +1110,3 @@ def test_scalar_setitem_series_with_nested_value_length1(value, indexer_sli): assert (ser.loc[0] == value).all() else: assert ser.loc[0] == value - - -class TestSetitemValidation: - # This is adapted from pandas/tests/arrays/masked/test_indexing.py - def _check_setitem_invalid(self, arr, invalid): - msg = "Setting an item of incompatible dtype is deprecated" - msg = re.escape(msg) - with tm.assert_produces_warning(FutureWarning, match=msg): - arr[0] = invalid - - # FIXME: don't leave commented-out - # with tm.assert_produces_warning(FutureWarning, match=msg): - # arr[:] = invalid - - # with tm.assert_produces_warning(FutureWarning, match=msg): - # arr[[0]] = invalid - - # with pytest.raises(TypeError): - # arr[[0]] = [invalid] - - # with pytest.raises(TypeError): - # arr[[0]] = np.array([invalid], dtype=object) - - _invalid_scalars = [ - 1 + 2j, - "True", - "1", - "1.0", - NaT, - np.datetime64("NaT"), - np.timedelta64("NaT"), - ] - - @pytest.mark.parametrize( - "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] - ) - def test_setitem_validation_scalar_bool(self, invalid): - arr = Series([True, False, None], dtype="bool") - self._check_setitem_invalid(arr, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) - def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): - arr = Series([1, 2, 3], dtype=any_int_numpy_dtype) - self._check_setitem_invalid(arr, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) - def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): - arr = Series([1, 2, None], dtype=float_numpy_dtype) - self._check_setitem_invalid(arr, invalid) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index b347691c3c101..a9defbbaa525f 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -13,6 +13,7 @@ Index, IndexSlice, MultiIndex, + NaT, Series, Timedelta, Timestamp, @@ -437,3 +438,52 @@ def test_setitem_dict_and_set_disallowed_multiindex(self, key): ser = Series([1, 2], index=MultiIndex.from_tuples([(1, 2), (3, 4)])) with pytest.raises(TypeError, match="as an indexer is not supported"): ser.loc[key] = 1 + + +class TestSetitemValidation: + # This is adapted from pandas/tests/arrays/masked/test_indexing.py + def _check_setitem_invalid(self, arr, invalid): + msg = "Setting an item of incompatible dtype is deprecated" + msg = re.escape(msg) + with tm.assert_produces_warning(FutureWarning, match=msg): + arr[0] = invalid + + # FIXME: don't leave commented-out + # with tm.assert_produces_warning(FutureWarning, match=msg): + # arr[:] = invalid + + # with tm.assert_produces_warning(FutureWarning, match=msg): + # arr[[0]] = invalid + + # with pytest.raises(TypeError): + # arr[[0]] = [invalid] + + # with pytest.raises(TypeError): + # arr[[0]] = np.array([invalid], dtype=object) + + _invalid_scalars = [ + 1 + 2j, + "True", + "1", + "1.0", + NaT, + np.datetime64("NaT"), + np.timedelta64("NaT"), + ] + + @pytest.mark.parametrize( + "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] + ) + def test_setitem_validation_scalar_bool(self, invalid): + arr = Series([True, False, None], dtype="bool") + self._check_setitem_invalid(arr, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) + def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): + arr = Series([1, 2, 3], dtype=any_int_numpy_dtype) + self._check_setitem_invalid(arr, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + arr = Series([1, 2, None], dtype=float_numpy_dtype) + self._check_setitem_invalid(arr, invalid) From d16eea6134ea3249d7adef5811c6e2be998a5fa4 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Fri, 12 May 2023 16:15:06 +0200 Subject: [PATCH 68/80] add tests about warnings --- pandas/tests/indexing/test_indexing.py | 84 +++++++++++++++++++------- 1 file changed, 63 insertions(+), 21 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 5c5f13e417842..1fab757e16ebb 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1117,22 +1117,55 @@ class TestSetitemValidation: def _check_setitem_invalid(self, arr, invalid): msg = "Setting an item of incompatible dtype is deprecated" msg = re.escape(msg) + + orig_arr = arr.copy() + + # setitem with tm.assert_produces_warning(FutureWarning, match=msg): arr[0] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr[[0]] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr[0:1] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr[[True, False, False]] = invalid + arr = orig_arr.copy() + + # iloc + with tm.assert_produces_warning(FutureWarning, match=msg): + arr.iloc[0] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr.iloc[[0]] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr.iloc[0:1] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr.iloc[[True, False, False]] = invalid + arr = orig_arr.copy() + + # loc + with tm.assert_produces_warning(FutureWarning, match=msg): + arr.loc[0] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr.loc[[0]] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr.loc[0:1] = invalid + arr = orig_arr.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + arr.loc[[True, False, False]] = invalid + arr = orig_arr.copy() # FIXME: don't leave commented-out # with tm.assert_produces_warning(FutureWarning, match=msg): # arr[:] = invalid - # with tm.assert_produces_warning(FutureWarning, match=msg): - # arr[[0]] = invalid - - # with pytest.raises(TypeError): - # arr[[0]] = [invalid] - - # with pytest.raises(TypeError): - # arr[[0]] = np.array([invalid], dtype=object) - _invalid_scalars = [ 1 + 2j, "True", @@ -1143,19 +1176,28 @@ def _check_setitem_invalid(self, arr, invalid): np.timedelta64("NaT"), ] - @pytest.mark.parametrize( - "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] - ) - def test_setitem_validation_scalar_bool(self, invalid): - arr = Series([True, False, None], dtype="bool") - self._check_setitem_invalid(arr, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) + # @pytest.mark.parametrize( + # "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] + # ) + # @pytest.mark.parametrize( + # "indexer", + # [ + # slice(0, 1), + # [True, False, False], + # 0, + # [0], + # ] + # ) + # def test_setitem_validation_scalar_bool(self, invalid, indexer): + # arr = Series([True, False], dtype="bool") + # self._check_setitem_invalid(arr, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [1.5, np.float64(1.5)]) def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): arr = Series([1, 2, 3], dtype=any_int_numpy_dtype) self._check_setitem_invalid(arr, invalid) - @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) - def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): - arr = Series([1, 2, None], dtype=float_numpy_dtype) - self._check_setitem_invalid(arr, invalid) + # @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) + # def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + # arr = Series([1, 2, None], dtype=float_numpy_dtype) + # self._check_setitem_invalid(arr, invalid) From 25198d48fe90c1aa5118af857a162b8c1eda5c4a Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Fri, 12 May 2023 16:25:14 +0200 Subject: [PATCH 69/80] fixup tests --- pandas/tests/indexing/test_indexing.py | 33 ++++++++++---------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 1fab757e16ebb..8c41021083459 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1176,28 +1176,19 @@ def _check_setitem_invalid(self, arr, invalid): np.timedelta64("NaT"), ] - # @pytest.mark.parametrize( - # "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] - # ) - # @pytest.mark.parametrize( - # "indexer", - # [ - # slice(0, 1), - # [True, False, False], - # 0, - # [0], - # ] - # ) - # def test_setitem_validation_scalar_bool(self, invalid, indexer): - # arr = Series([True, False], dtype="bool") - # self._check_setitem_invalid(arr, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [1.5, np.float64(1.5)]) + @pytest.mark.parametrize( + "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] + ) + def test_setitem_validation_scalar_bool(self, invalid): + arr = Series([True, False, False], dtype="bool") + self._check_setitem_invalid(arr, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): arr = Series([1, 2, 3], dtype=any_int_numpy_dtype) self._check_setitem_invalid(arr, invalid) - # @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) - # def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): - # arr = Series([1, 2, None], dtype=float_numpy_dtype) - # self._check_setitem_invalid(arr, invalid) + @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + arr = Series([1, 2, None], dtype=float_numpy_dtype) + self._check_setitem_invalid(arr, invalid) From ba6daa5bb550d51a26e3fe16719cb22005fc1db4 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Fri, 12 May 2023 16:37:50 +0200 Subject: [PATCH 70/80] add dataframe tests too --- pandas/tests/indexing/test_indexing.py | 131 ++++++++++++++++++------- 1 file changed, 98 insertions(+), 33 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 8c41021083459..a8f267990dc41 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1114,57 +1114,58 @@ def test_scalar_setitem_series_with_nested_value_length1(value, indexer_sli): class TestSetitemValidation: # This is adapted from pandas/tests/arrays/masked/test_indexing.py - def _check_setitem_invalid(self, arr, invalid): + def _check_setitem_invalid(self, ser, invalid): msg = "Setting an item of incompatible dtype is deprecated" msg = re.escape(msg) - orig_arr = arr.copy() + orig_ser = ser.copy() # setitem with tm.assert_produces_warning(FutureWarning, match=msg): - arr[0] = invalid - arr = orig_arr.copy() + ser[0] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr[[0]] = invalid - arr = orig_arr.copy() + ser[[0]] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr[0:1] = invalid - arr = orig_arr.copy() + ser[0:1] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr[[True, False, False]] = invalid - arr = orig_arr.copy() + ser[[True, False, False]] = invalid + ser = orig_ser.copy() # iloc with tm.assert_produces_warning(FutureWarning, match=msg): - arr.iloc[0] = invalid - arr = orig_arr.copy() + ser.iloc[0] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr.iloc[[0]] = invalid - arr = orig_arr.copy() + ser.iloc[[0]] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr.iloc[0:1] = invalid - arr = orig_arr.copy() + ser.iloc[0:1] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr.iloc[[True, False, False]] = invalid - arr = orig_arr.copy() + ser.iloc[[True, False, False]] = invalid + ser = orig_ser.copy() # loc with tm.assert_produces_warning(FutureWarning, match=msg): - arr.loc[0] = invalid - arr = orig_arr.copy() + ser.loc[0] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr.loc[[0]] = invalid - arr = orig_arr.copy() + ser.loc[[0]] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr.loc[0:1] = invalid - arr = orig_arr.copy() + ser.loc[0:1] = invalid + ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - arr.loc[[True, False, False]] = invalid - arr = orig_arr.copy() + ser.loc[[True, False, False]] = invalid + ser = orig_ser.copy() + # note: commented-out in the EA case too # FIXME: don't leave commented-out # with tm.assert_produces_warning(FutureWarning, match=msg): - # arr[:] = invalid + # ser[:] = invalid _invalid_scalars = [ 1 + 2j, @@ -1180,15 +1181,79 @@ def _check_setitem_invalid(self, arr, invalid): "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] ) def test_setitem_validation_scalar_bool(self, invalid): - arr = Series([True, False, False], dtype="bool") - self._check_setitem_invalid(arr, invalid) + ser = Series([True, False, False], dtype="bool") + self._check_setitem_invalid(ser, invalid) @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): - arr = Series([1, 2, 3], dtype=any_int_numpy_dtype) - self._check_setitem_invalid(arr, invalid) + ser = Series([1, 2, 3], dtype=any_int_numpy_dtype) + self._check_setitem_invalid(ser, invalid) @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): - arr = Series([1, 2, None], dtype=float_numpy_dtype) - self._check_setitem_invalid(arr, invalid) + ser = Series([1, 2, None], dtype=float_numpy_dtype) + self._check_setitem_invalid(ser, invalid) + + +class TestSetitemValidationDataFrame: + # This is adapted from pandas/tests/arrays/masked/test_indexing.py + def _check_setitem_invalid(self, df, invalid): + msg = "Setting an item of incompatible dtype is deprecated" + msg = re.escape(msg) + + orig_df = df.copy() + + # iloc + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[0, 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[[0], 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[0:1, 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[[True, False, False], 0] = invalid + df = orig_df.copy() + + # loc + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[0, "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[[0], "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[0:1, "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[[True, False, False], "a"] = invalid + df = orig_df.copy() + + _invalid_scalars = [ + 1 + 2j, + "True", + "1", + "1.0", + NaT, + np.datetime64("NaT"), + np.timedelta64("NaT"), + ] + + @pytest.mark.parametrize( + "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] + ) + def test_setitem_validation_scalar_bool(self, invalid): + df = DataFrame({"a": [True, False, False]}, dtype="bool") + self._check_setitem_invalid(df, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) + def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): + df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype) + self._check_setitem_invalid(df, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + df = DataFrame({"a": [1, 2, None]}, dtype=float_numpy_dtype) + self._check_setitem_invalid(df, invalid) From 9b15bc2dc5b88444a812a98635efa6741e5097cd Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Fri, 12 May 2023 16:49:59 +0200 Subject: [PATCH 71/80] fixup tests --- pandas/tests/frame/indexing/test_indexing.py | 65 ++++++++ pandas/tests/series/indexing/test_indexing.py | 150 ++++++++++++++++++ 2 files changed, 215 insertions(+) diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index a985acd8c003f..fd2e8bf7899eb 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -1870,3 +1870,68 @@ def test_setitem_dict_and_set_disallowed_multiindex(self, key): ) with pytest.raises(TypeError, match="as an indexer is not supported"): df.loc[key] = 1 + + +class TestSetitemValidation: + # This is adapted from pandas/tests/arrays/masked/test_indexing.py + # but checks for warnings instead of errors. + def _check_setitem_invalid(self, df, invalid): + msg = "Setting an item of incompatible dtype is deprecated" + msg = re.escape(msg) + + orig_df = df.copy() + + # iloc + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[0, 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[[0], 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[0:1, 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[[True, False, False], 0] = invalid + df = orig_df.copy() + + # loc + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[0, "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[[0], "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[0:1, "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[[True, False, False], "a"] = invalid + df = orig_df.copy() + + _invalid_scalars = [ + 1 + 2j, + "True", + "1", + "1.0", + pd.NaT, + np.datetime64("NaT"), + np.timedelta64("NaT"), + ] + + @pytest.mark.parametrize( + "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] + ) + def test_setitem_validation_scalar_bool(self, invalid): + df = DataFrame({"a": [True, False, False]}, dtype="bool") + self._check_setitem_invalid(df, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) + def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): + df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype) + self._check_setitem_invalid(df, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + df = DataFrame({"a": [1, 2, None]}, dtype=float_numpy_dtype) + self._check_setitem_invalid(df, invalid) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index b347691c3c101..76a4448b20563 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -13,6 +13,7 @@ Index, IndexSlice, MultiIndex, + NaT, Series, Timedelta, Timestamp, @@ -437,3 +438,152 @@ def test_setitem_dict_and_set_disallowed_multiindex(self, key): ser = Series([1, 2], index=MultiIndex.from_tuples([(1, 2), (3, 4)])) with pytest.raises(TypeError, match="as an indexer is not supported"): ser.loc[key] = 1 + + +class TestSetitemValidation: + # This is adapted from pandas/tests/arrays/masked/test_indexing.py + # but checks for warnings instead of errors. + def _check_setitem_invalid(self, ser, invalid): + msg = "Setting an item of incompatible dtype is deprecated" + msg = re.escape(msg) + + orig_ser = ser.copy() + + # setitem + with tm.assert_produces_warning(FutureWarning, match=msg): + ser[0] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser[[0]] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser[0:1] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser[[True, False, False]] = invalid + ser = orig_ser.copy() + + # iloc + with tm.assert_produces_warning(FutureWarning, match=msg): + ser.iloc[0] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser.iloc[[0]] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser.iloc[0:1] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser.iloc[[True, False, False]] = invalid + ser = orig_ser.copy() + + # loc + with tm.assert_produces_warning(FutureWarning, match=msg): + ser.loc[0] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser.loc[[0]] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser.loc[0:1] = invalid + ser = orig_ser.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + ser.loc[[True, False, False]] = invalid + ser = orig_ser.copy() + + # note: commented-out in the EA case too + # FIXME: don't leave commented-out + # with tm.assert_produces_warning(FutureWarning, match=msg): + # ser[:] = invalid + + _invalid_scalars = [ + 1 + 2j, + "True", + "1", + "1.0", + NaT, + np.datetime64("NaT"), + np.timedelta64("NaT"), + ] + + @pytest.mark.parametrize( + "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] + ) + def test_setitem_validation_scalar_bool(self, invalid): + ser = Series([True, False, False], dtype="bool") + self._check_setitem_invalid(ser, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) + def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): + ser = Series([1, 2, 3], dtype=any_int_numpy_dtype) + self._check_setitem_invalid(ser, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + ser = Series([1, 2, None], dtype=float_numpy_dtype) + self._check_setitem_invalid(ser, invalid) + + +class TestSetitemValidationDataFrame: + # This is adapted from pandas/tests/arrays/masked/test_indexing.py + # but checks for warnings instead of errors. + def _check_setitem_invalid(self, df, invalid): + msg = "Setting an item of incompatible dtype is deprecated" + msg = re.escape(msg) + + orig_df = df.copy() + + # iloc + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[0, 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[[0], 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[0:1, 0] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[[True, False, False], 0] = invalid + df = orig_df.copy() + + # loc + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[0, "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[[0], "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[0:1, "a"] = invalid + df = orig_df.copy() + with tm.assert_produces_warning(FutureWarning, match=msg): + df.loc[[True, False, False], "a"] = invalid + df = orig_df.copy() + + _invalid_scalars = [ + 1 + 2j, + "True", + "1", + "1.0", + NaT, + np.datetime64("NaT"), + np.timedelta64("NaT"), + ] + + @pytest.mark.parametrize( + "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] + ) + def test_setitem_validation_scalar_bool(self, invalid): + df = DataFrame({"a": [True, False, False]}, dtype="bool") + self._check_setitem_invalid(df, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) + def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): + df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype) + self._check_setitem_invalid(df, invalid) + + @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + df = DataFrame({"a": [1, 2, None]}, dtype=float_numpy_dtype) + self._check_setitem_invalid(df, invalid) From ebd1a509f8ed94f5d1c075e5f4466754dc1f2704 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Fri, 12 May 2023 17:07:33 +0200 Subject: [PATCH 72/80] simplify --- pandas/tests/frame/indexing/test_indexing.py | 40 ++--- pandas/tests/indexing/test_indexing.py | 147 ------------------ pandas/tests/series/indexing/test_indexing.py | 119 ++------------ 3 files changed, 27 insertions(+), 279 deletions(-) diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index fd2e8bf7899eb..499ef5654a2ce 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -1875,7 +1875,7 @@ def test_setitem_dict_and_set_disallowed_multiindex(self, key): class TestSetitemValidation: # This is adapted from pandas/tests/arrays/masked/test_indexing.py # but checks for warnings instead of errors. - def _check_setitem_invalid(self, df, invalid): + def _check_setitem_invalid(self, df, invalid, indexer): msg = "Setting an item of incompatible dtype is deprecated" msg = re.escape(msg) @@ -1883,30 +1883,12 @@ def _check_setitem_invalid(self, df, invalid): # iloc with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[0, 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[[0], 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[0:1, 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[[True, False, False], 0] = invalid + df.iloc[indexer, 0] = invalid df = orig_df.copy() # loc with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[0, "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[[0], "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[0:1, "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[[True, False, False], "a"] = invalid + df.loc[indexer, "a"] = invalid df = orig_df.copy() _invalid_scalars = [ @@ -1918,20 +1900,24 @@ def _check_setitem_invalid(self, df, invalid): np.datetime64("NaT"), np.timedelta64("NaT"), ] + _indexers = [0, [0], slice(0, 1), [True, False, False]] @pytest.mark.parametrize( "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] ) - def test_setitem_validation_scalar_bool(self, invalid): + @pytest.mark.parametrize("indexer", _indexers) + def test_setitem_validation_scalar_bool(self, invalid, indexer): df = DataFrame({"a": [True, False, False]}, dtype="bool") - self._check_setitem_invalid(df, invalid) + self._check_setitem_invalid(df, invalid, indexer) @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) - def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): + @pytest.mark.parametrize("indexer", _indexers) + def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype, indexer): df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype) - self._check_setitem_invalid(df, invalid) + self._check_setitem_invalid(df, invalid, indexer) @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) - def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + @pytest.mark.parametrize("indexer", _indexers) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype, indexer): df = DataFrame({"a": [1, 2, None]}, dtype=float_numpy_dtype) - self._check_setitem_invalid(df, invalid) + self._check_setitem_invalid(df, invalid, indexer) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index a8f267990dc41..28f9b1a62172e 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1110,150 +1110,3 @@ def test_scalar_setitem_series_with_nested_value_length1(value, indexer_sli): assert (ser.loc[0] == value).all() else: assert ser.loc[0] == value - - -class TestSetitemValidation: - # This is adapted from pandas/tests/arrays/masked/test_indexing.py - def _check_setitem_invalid(self, ser, invalid): - msg = "Setting an item of incompatible dtype is deprecated" - msg = re.escape(msg) - - orig_ser = ser.copy() - - # setitem - with tm.assert_produces_warning(FutureWarning, match=msg): - ser[0] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser[[0]] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser[0:1] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser[[True, False, False]] = invalid - ser = orig_ser.copy() - - # iloc - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.iloc[0] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.iloc[[0]] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.iloc[0:1] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.iloc[[True, False, False]] = invalid - ser = orig_ser.copy() - - # loc - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.loc[0] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.loc[[0]] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.loc[0:1] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.loc[[True, False, False]] = invalid - ser = orig_ser.copy() - - # note: commented-out in the EA case too - # FIXME: don't leave commented-out - # with tm.assert_produces_warning(FutureWarning, match=msg): - # ser[:] = invalid - - _invalid_scalars = [ - 1 + 2j, - "True", - "1", - "1.0", - NaT, - np.datetime64("NaT"), - np.timedelta64("NaT"), - ] - - @pytest.mark.parametrize( - "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] - ) - def test_setitem_validation_scalar_bool(self, invalid): - ser = Series([True, False, False], dtype="bool") - self._check_setitem_invalid(ser, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) - def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): - ser = Series([1, 2, 3], dtype=any_int_numpy_dtype) - self._check_setitem_invalid(ser, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) - def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): - ser = Series([1, 2, None], dtype=float_numpy_dtype) - self._check_setitem_invalid(ser, invalid) - - -class TestSetitemValidationDataFrame: - # This is adapted from pandas/tests/arrays/masked/test_indexing.py - def _check_setitem_invalid(self, df, invalid): - msg = "Setting an item of incompatible dtype is deprecated" - msg = re.escape(msg) - - orig_df = df.copy() - - # iloc - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[0, 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[[0], 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[0:1, 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[[True, False, False], 0] = invalid - df = orig_df.copy() - - # loc - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[0, "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[[0], "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[0:1, "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[[True, False, False], "a"] = invalid - df = orig_df.copy() - - _invalid_scalars = [ - 1 + 2j, - "True", - "1", - "1.0", - NaT, - np.datetime64("NaT"), - np.timedelta64("NaT"), - ] - - @pytest.mark.parametrize( - "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] - ) - def test_setitem_validation_scalar_bool(self, invalid): - df = DataFrame({"a": [True, False, False]}, dtype="bool") - self._check_setitem_invalid(df, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) - def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): - df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype) - self._check_setitem_invalid(df, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) - def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): - df = DataFrame({"a": [1, 2, None]}, dtype=float_numpy_dtype) - self._check_setitem_invalid(df, invalid) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 76a4448b20563..c68b2897076b8 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -443,52 +443,22 @@ def test_setitem_dict_and_set_disallowed_multiindex(self, key): class TestSetitemValidation: # This is adapted from pandas/tests/arrays/masked/test_indexing.py # but checks for warnings instead of errors. - def _check_setitem_invalid(self, ser, invalid): + def _check_setitem_invalid(self, ser, invalid, indexer): msg = "Setting an item of incompatible dtype is deprecated" msg = re.escape(msg) orig_ser = ser.copy() - # setitem with tm.assert_produces_warning(FutureWarning, match=msg): - ser[0] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser[[0]] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser[0:1] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser[[True, False, False]] = invalid + ser[indexer] = invalid ser = orig_ser.copy() - # iloc - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.iloc[0] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.iloc[[0]] = invalid - ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - ser.iloc[0:1] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.iloc[[True, False, False]] = invalid + ser.iloc[indexer] = invalid ser = orig_ser.copy() - # loc - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.loc[0] = invalid - ser = orig_ser.copy() with tm.assert_produces_warning(FutureWarning, match=msg): - ser.loc[[0]] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.loc[0:1] = invalid - ser = orig_ser.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - ser.loc[[True, False, False]] = invalid + ser.loc[indexer] = invalid ser = orig_ser.copy() # note: commented-out in the EA case too @@ -505,85 +475,24 @@ def _check_setitem_invalid(self, ser, invalid): np.datetime64("NaT"), np.timedelta64("NaT"), ] + _indexers = [0, [0], slice(0, 1), [True, False, False]] @pytest.mark.parametrize( "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] ) - def test_setitem_validation_scalar_bool(self, invalid): + @pytest.mark.parametrize("indexer", _indexers) + def test_setitem_validation_scalar_bool(self, invalid, indexer): ser = Series([True, False, False], dtype="bool") - self._check_setitem_invalid(ser, invalid) + self._check_setitem_invalid(ser, invalid, indexer) @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) - def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): + @pytest.mark.parametrize("indexer", _indexers) + def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype, indexer): ser = Series([1, 2, 3], dtype=any_int_numpy_dtype) - self._check_setitem_invalid(ser, invalid) + self._check_setitem_invalid(ser, invalid, indexer) @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) - def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): + @pytest.mark.parametrize("indexer", _indexers) + def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype, indexer): ser = Series([1, 2, None], dtype=float_numpy_dtype) - self._check_setitem_invalid(ser, invalid) - - -class TestSetitemValidationDataFrame: - # This is adapted from pandas/tests/arrays/masked/test_indexing.py - # but checks for warnings instead of errors. - def _check_setitem_invalid(self, df, invalid): - msg = "Setting an item of incompatible dtype is deprecated" - msg = re.escape(msg) - - orig_df = df.copy() - - # iloc - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[0, 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[[0], 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[0:1, 0] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.iloc[[True, False, False], 0] = invalid - df = orig_df.copy() - - # loc - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[0, "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[[0], "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[0:1, "a"] = invalid - df = orig_df.copy() - with tm.assert_produces_warning(FutureWarning, match=msg): - df.loc[[True, False, False], "a"] = invalid - df = orig_df.copy() - - _invalid_scalars = [ - 1 + 2j, - "True", - "1", - "1.0", - NaT, - np.datetime64("NaT"), - np.timedelta64("NaT"), - ] - - @pytest.mark.parametrize( - "invalid", _invalid_scalars + [1, 1.0, np.int64(1), np.float64(1)] - ) - def test_setitem_validation_scalar_bool(self, invalid): - df = DataFrame({"a": [True, False, False]}, dtype="bool") - self._check_setitem_invalid(df, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True, 1.5, np.float64(1.5)]) - def test_setitem_validation_scalar_int(self, invalid, any_int_numpy_dtype): - df = DataFrame({"a": [1, 2, 3]}, dtype=any_int_numpy_dtype) - self._check_setitem_invalid(df, invalid) - - @pytest.mark.parametrize("invalid", _invalid_scalars + [True]) - def test_setitem_validation_scalar_float(self, invalid, float_numpy_dtype): - df = DataFrame({"a": [1, 2, None]}, dtype=float_numpy_dtype) - self._check_setitem_invalid(df, invalid) + self._check_setitem_invalid(ser, invalid, indexer) From a83528134edbff4f4677fb53ccd77c93d71cc2a3 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Fri, 12 May 2023 17:08:54 +0200 Subject: [PATCH 73/80] try-fix docs build --- doc/source/whatsnew/v0.21.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.21.0.rst b/doc/source/whatsnew/v0.21.0.rst index 90fdd6720fb07..1dae2e8463c27 100644 --- a/doc/source/whatsnew/v0.21.0.rst +++ b/doc/source/whatsnew/v0.21.0.rst @@ -754,7 +754,7 @@ Previously assignments, ``.where()`` and ``.fillna()`` with a ``bool`` assignmen New behavior -.. codeblock:: ipython +.. code-block:: ipython In [7]: s[1] = True From 5ee4ebfe7afec34a5e819912e96fbd68c1a4aa55 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 22 May 2023 13:18:01 +0100 Subject: [PATCH 74/80] post-merge fixup --- pandas/tests/frame/test_constructors.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index 0999968477a81..b5616c9f7dc49 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -2556,17 +2556,17 @@ def check_views(c_only: bool = False): # TODO: most of the rest of this test belongs in indexing tests if ( - is_float_dtype(df.dtypes[0]) - or is_integer_dtype(df.dtypes[0]) - or is_complex_dtype(df.dtypes[0]) - or is_object_dtype(df.dtypes[0]) - or is_datetime64_dtype(df.dtypes[0]) # TODO this one should warn - or is_timedelta64_dtype(df.dtypes[0]) # TODO this one should warn + is_float_dtype(df.dtypes.iloc[0]) + or is_integer_dtype(df.dtypes.iloc[0]) + or is_complex_dtype(df.dtypes.iloc[0]) + or is_object_dtype(df.dtypes.iloc[0]) + or is_datetime64_dtype(df.dtypes.iloc[0]) # TODO this one should warn + or is_timedelta64_dtype(df.dtypes.iloc[0]) # TODO this one should warn ): warn = None else: warn = FutureWarning - with tm.assert_produces_warning(warn): + with tm.assert_produces_warning(warn, match="incompatible dtype"): df.iloc[0, 0] = 0 df.iloc[0, 1] = 0 if not copy: From bb37b0922f9257b6b80721a2ff9673b4b5ec86c0 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 22 May 2023 13:21:04 +0100 Subject: [PATCH 75/80] raise assertionerror if self.dtype equals new_dtype --- pandas/core/internals/blocks.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 544f51ef3d45c..3fb62979922ef 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -425,7 +425,7 @@ def coerce_to_target_dtype(self, other, warn_on_upcast: bool = False) -> Block: and will receive the same block """ new_dtype = find_result_type(self.values, other) - if warn_on_upcast and self.dtype != new_dtype: + if warn_on_upcast: warnings.warn( f"Setting an item of incompatible dtype is deprecated " "and will raise in a future error of pandas. " @@ -434,6 +434,12 @@ def coerce_to_target_dtype(self, other, warn_on_upcast: bool = False) -> Block: FutureWarning, stacklevel=find_stack_level(), ) + if self.dtype == new_dtype: + raise AssertionError( + f"Did not expect new dtype {new_dtype} to equal self.dtype " + f"{self.dtype}. Please report a bug at " + "https://github.com/pandas-dev/pandas/issues." + ) return self.astype(new_dtype, copy=False) @final From 7df15e9017d18c1d14187bfaf897795ba5e77229 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 22 May 2023 13:25:01 +0100 Subject: [PATCH 76/80] add todo for test case which should warn --- pandas/tests/frame/indexing/test_indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index a9dcb6bc03d2c..6b4ab5e324a8d 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -1343,7 +1343,7 @@ def test_loc_expand_empty_frame_keep_midx_names(self): [ ("x", "a", None), ("x", ["a"], None), - (1, "a", None), + (1, "a", None), # todo: this one should warn as well (1, ["a"], FutureWarning), ], ) From f323d2a0706c7d169798b86d6e5129e83b6b2428 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 22 May 2023 13:35:43 +0100 Subject: [PATCH 77/80] add more todos --- pandas/tests/frame/indexing/test_indexing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index 6b4ab5e324a8d..2e90f816ae760 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -1341,9 +1341,9 @@ def test_loc_expand_empty_frame_keep_midx_names(self): @pytest.mark.parametrize( "val, idxr, warn", [ - ("x", "a", None), - ("x", ["a"], None), - (1, "a", None), # todo: this one should warn as well + ("x", "a", None), # TODO: this should warn as well + ("x", ["a"], None), # TODO: this should warn as well + (1, "a", None), # TODO: this should warn as well (1, ["a"], FutureWarning), ], ) From 5f5a6a5cee8dc4ed7ee27a7c98c5cb6488790564 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 22 May 2023 14:35:43 +0100 Subject: [PATCH 78/80] post-merge fixup --- pandas/tests/series/indexing/test_indexing.py | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index c68b2897076b8..59dfc218f8f6b 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -28,11 +28,14 @@ def test_basic_indexing(): s = Series(np.random.randn(5), index=["a", "b", "a", "a", "b"]) + warn_msg = "Series.__[sg]etitem__ treating keys as positions is deprecated" msg = "index 5 is out of bounds for axis 0 with size 5" with pytest.raises(IndexError, match=msg): - s[5] + with tm.assert_produces_warning(FutureWarning, match=warn_msg): + s[5] with pytest.raises(IndexError, match=msg): - s[5] = 0 + with tm.assert_produces_warning(FutureWarning, match=warn_msg): + s[5] = 0 with pytest.raises(KeyError, match=r"^'c'$"): s["c"] @@ -40,10 +43,12 @@ def test_basic_indexing(): s = s.sort_index() with pytest.raises(IndexError, match=msg): - s[5] + with tm.assert_produces_warning(FutureWarning, match=warn_msg): + s[5] msg = r"index 5 is out of bounds for axis (0|1) with size 5|^5$" with pytest.raises(IndexError, match=msg): - s[5] = 0 + with tm.assert_produces_warning(FutureWarning, match=warn_msg): + s[5] = 0 def test_getitem_numeric_should_not_fallback_to_positional(any_numeric_dtype): @@ -145,7 +150,9 @@ def test_series_box_timestamp(): assert isinstance(ser.iloc[4], Timestamp) ser = Series(rng, index=rng) - assert isinstance(ser[0], Timestamp) + msg = "Series.__getitem__ treating keys as positions is deprecated" + with tm.assert_produces_warning(FutureWarning, match=msg): + assert isinstance(ser[0], Timestamp) assert isinstance(ser.at[rng[1]], Timestamp) assert isinstance(ser.iat[2], Timestamp) assert isinstance(ser.loc[rng[3]], Timestamp) @@ -188,12 +195,12 @@ def test_setitem_ambiguous_keyerror(indexer_sl): def test_setitem(datetime_series): datetime_series[datetime_series.index[5]] = np.NaN - datetime_series[[1, 2, 17]] = np.NaN - datetime_series[6] = np.NaN - assert np.isnan(datetime_series[6]) - assert np.isnan(datetime_series[2]) + datetime_series.iloc[[1, 2, 17]] = np.NaN + datetime_series.iloc[6] = np.NaN + assert np.isnan(datetime_series.iloc[6]) + assert np.isnan(datetime_series.iloc[2]) datetime_series[np.isnan(datetime_series)] = 5 - assert not np.isnan(datetime_series[2]) + assert not np.isnan(datetime_series.iloc[2]) def test_setslice(datetime_series): @@ -292,9 +299,9 @@ def test_underlying_data_conversion(using_copy_on_write): def test_preserve_refs(datetime_series): - seq = datetime_series[[5, 10, 15]] - seq[1] = np.NaN - assert not np.isnan(datetime_series[10]) + seq = datetime_series.iloc[[5, 10, 15]] + seq.iloc[1] = np.NaN + assert not np.isnan(datetime_series.iloc[10]) def test_multilevel_preserve_name(lexsorted_two_level_string_multiindex, indexer_sl): From 0381c62cd6a0d5fdad253161bc92cfc2a1f0f18b Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 3 Jul 2023 09:39:47 +0100 Subject: [PATCH 79/80] move logic for determining warning into the test --- pandas/tests/series/indexing/test_where.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pandas/tests/series/indexing/test_where.py b/pandas/tests/series/indexing/test_where.py index 4e15d6de4a32f..4de3b64294504 100644 --- a/pandas/tests/series/indexing/test_where.py +++ b/pandas/tests/series/indexing/test_where.py @@ -38,22 +38,27 @@ def test_where_unsafe_float(float_numpy_dtype): @pytest.mark.parametrize( - "dtype,expected_dtype,warn", + "dtype,expected_dtype", [ - (np.int8, np.float64, FutureWarning), - (np.int16, np.float64, FutureWarning), - (np.int32, np.float64, FutureWarning), - (np.int64, np.float64, FutureWarning), - (np.float32, np.float32, None), - (np.float64, np.float64, None), + (np.int8, np.float64), + (np.int16, np.float64), + (np.int32, np.float64), + (np.int64, np.float64), + (np.float32, np.float32), + (np.float64, np.float64), ], ) -def test_where_unsafe_upcast(dtype, expected_dtype, warn): +def test_where_unsafe_upcast(dtype, expected_dtype): # see gh-9743 s = Series(np.arange(10), dtype=dtype) values = [2.5, 3.5, 4.5, 5.5, 6.5] mask = s < 5 expected = Series(values + list(range(5, 10)), dtype=expected_dtype) + warn = ( + None + if np.dtype(dtype).kind == np.dtype(expected_dtype).kind == "f" + else FutureWarning + ) with tm.assert_produces_warning(warn, match="incompatible dtype"): s[mask] = values tm.assert_series_equal(s, expected) From a843e305d869e17c70c4749ba5d8da7070c19c17 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:48:02 +0100 Subject: [PATCH 80/80] uncomment test --- pandas/tests/frame/test_constructors.py | 20 ++++--------------- pandas/tests/series/indexing/test_indexing.py | 6 ++---- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/pandas/tests/frame/test_constructors.py b/pandas/tests/frame/test_constructors.py index a790b24b27ff0..31f404258a9bb 100644 --- a/pandas/tests/frame/test_constructors.py +++ b/pandas/tests/frame/test_constructors.py @@ -23,17 +23,11 @@ import pytest import pytz +from pandas._libs import lib from pandas.errors import IntCastingNaNError import pandas.util._test_decorators as td -from pandas.core.dtypes.common import ( - is_complex_dtype, - is_datetime64_dtype, - is_float_dtype, - is_integer_dtype, - is_object_dtype, - is_timedelta64_dtype, -) +from pandas.core.dtypes.common import is_integer_dtype from pandas.core.dtypes.dtypes import ( DatetimeTZDtype, IntervalDtype, @@ -2555,14 +2549,8 @@ def check_views(c_only: bool = False): check_views() # TODO: most of the rest of this test belongs in indexing tests - if ( - is_float_dtype(df.dtypes.iloc[0]) - or is_integer_dtype(df.dtypes.iloc[0]) - or is_complex_dtype(df.dtypes.iloc[0]) - or is_object_dtype(df.dtypes.iloc[0]) - or is_datetime64_dtype(df.dtypes.iloc[0]) # TODO this one should warn - or is_timedelta64_dtype(df.dtypes.iloc[0]) # TODO this one should warn - ): + # TODO: 'm' and 'M' should warn + if lib.is_np_dtype(df.dtypes.iloc[0], "fciuOmM"): warn = None else: warn = FutureWarning diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index aa0a46f057c29..aacbcc59068bd 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -470,10 +470,8 @@ def _check_setitem_invalid(self, ser, invalid, indexer): ser.loc[indexer] = invalid ser = orig_ser.copy() - # note: commented-out in the EA case too - # FIXME: don't leave commented-out - # with tm.assert_produces_warning(FutureWarning, match=msg): - # ser[:] = invalid + with tm.assert_produces_warning(FutureWarning, match=msg): + ser[:] = invalid _invalid_scalars = [ 1 + 2j,