From 205c0f0e3c88c1f02857571a1d5dccd0b41639af Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Wed, 24 Aug 2022 22:02:55 +0200 Subject: [PATCH 1/5] REGR: Fix regression RecursionError when replacing numeric scalar with None --- pandas/core/internals/blocks.py | 9 +++++++++ pandas/tests/frame/methods/test_replace.py | 11 +++++++++++ pandas/tests/series/methods/test_replace.py | 8 ++++++++ 3 files changed, 28 insertions(+) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 010358d3a21ec..8518c3624a4d1 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -608,6 +608,15 @@ def replace( blocks = [blk] return blocks + elif value is None and not self.is_object: + blk = self.astype(np.dtype(object)) + return blk.replace( + to_replace=to_replace, + value=value, + inplace=True, + mask=mask, + ) + elif self.ndim == 1 or self.shape[0] == 1: blk = self.coerce_to_target_dtype(value) return blk.replace( diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index 177f3ec1b4504..7a9d252520b88 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -1496,6 +1496,17 @@ def test_replace_list_with_mixed_type( result = obj.replace(box(to_replace), value) tm.assert_equal(result, expected) + @pytest.mark.parametrize("val", [2, np.nan, 2.0]) + def test_replace_value_none_dtype_numeric(self, val): + # GH#48231 + ser = DataFrame({"a": [1, val]}) + result = ser.replace(val, None) + expected = DataFrame({"a": [1, None]}, dtype="object") + tm.assert_frame_equal(result, expected) + + result = ser.replace({val: None}) + tm.assert_frame_equal(result, expected) + class TestDataFrameReplaceRegex: @pytest.mark.parametrize( diff --git a/pandas/tests/series/methods/test_replace.py b/pandas/tests/series/methods/test_replace.py index 77c9cf4013bd7..d17a0e5f2629e 100644 --- a/pandas/tests/series/methods/test_replace.py +++ b/pandas/tests/series/methods/test_replace.py @@ -667,3 +667,11 @@ def test_replace_different_int_types(self, any_int_numpy_dtype): result = labs.replace(map_dict) expected = labs.replace({0: 0, 2: 1, 1: 2}) tm.assert_series_equal(result, expected) + + @pytest.mark.parametrize("val", [2, np.nan, 2.0]) + def test_replace_value_none_dtype_numeric(self, val): + # GH#48231 + ser = pd.Series([1, val]) + result = ser.replace(val, None) + expected = pd.Series([1, None], dtype="object") + tm.assert_series_equal(result, expected) From 1d1e98c435c08f3ff92c91117069a165c7a01d39 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Fri, 7 Oct 2022 16:49:50 +0200 Subject: [PATCH 2/5] Update --- doc/source/whatsnew/v1.5.1.rst | 1 + pandas/core/internals/blocks.py | 10 +--------- pandas/tests/frame/methods/test_replace.py | 9 +++------ pandas/tests/series/methods/test_replace.py | 2 +- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/doc/source/whatsnew/v1.5.1.rst b/doc/source/whatsnew/v1.5.1.rst index 4d7576c013fd6..4b21c6c40a0b3 100644 --- a/doc/source/whatsnew/v1.5.1.rst +++ b/doc/source/whatsnew/v1.5.1.rst @@ -73,6 +73,7 @@ Fixed regressions - Fixed Regression in :meth:`DataFrame.loc` when setting values as a :class:`DataFrame` with all ``True`` indexer (:issue:`48701`) - Regression in :func:`.read_csv` causing an ``EmptyDataError`` when using an UTF-8 file handle that was already read from (:issue:`48646`) - Regression in :func:`to_datetime` when ``utc=True`` and ``arg`` contained timezone naive and aware arguments raised a ``ValueError`` (:issue:`48678`) +- Fixed regression in :meth:`Series.replace` raising ``RecursionError`` with numeric dtype and when specifying ``value=None`` (:issue:`45725`) - Fixed regression in :meth:`DataFrame.loc` raising ``FutureWarning`` when setting an empty :class:`DataFrame` (:issue:`48480`) - Fixed regression in :meth:`DataFrame.describe` raising ``TypeError`` when result contains ``NA`` (:issue:`48778`) - Fixed regression in :meth:`DataFrame.plot` ignoring invalid ``colormap`` for ``kind="scatter"`` (:issue:`48726`) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 330d0cbe14814..c4fea3eec7520 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -573,6 +573,7 @@ def replace( # Note: the checks we do in NDFrame.replace ensure we never get # here with listlike to_replace or value, as those cases # go through replace_list + value = self._standardize_fill_value(value) values = self.values @@ -611,15 +612,6 @@ def replace( blocks = [blk] return blocks - elif value is None and not self.is_object: - blk = self.astype(np.dtype(object)) - return blk.replace( - to_replace=to_replace, - value=value, - inplace=True, - mask=mask, - ) - elif self.ndim == 1 or self.shape[0] == 1: blk = self.coerce_to_target_dtype(value) return blk.replace( diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index 7a9d252520b88..f41cc709c9bf2 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -1499,12 +1499,9 @@ def test_replace_list_with_mixed_type( @pytest.mark.parametrize("val", [2, np.nan, 2.0]) def test_replace_value_none_dtype_numeric(self, val): # GH#48231 - ser = DataFrame({"a": [1, val]}) - result = ser.replace(val, None) - expected = DataFrame({"a": [1, None]}, dtype="object") - tm.assert_frame_equal(result, expected) - - result = ser.replace({val: None}) + df = DataFrame({"a": [1, val]}) + result = df.replace(val, None) + expected = DataFrame({"a": [1, np.nan]}) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/methods/test_replace.py b/pandas/tests/series/methods/test_replace.py index d17a0e5f2629e..3129c5f405379 100644 --- a/pandas/tests/series/methods/test_replace.py +++ b/pandas/tests/series/methods/test_replace.py @@ -673,5 +673,5 @@ def test_replace_value_none_dtype_numeric(self, val): # GH#48231 ser = pd.Series([1, val]) result = ser.replace(val, None) - expected = pd.Series([1, None], dtype="object") + expected = pd.Series([1, np.nan]) tm.assert_series_equal(result, expected) From 2402196fe2c119a27d91f65a091b8dcade59128e Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Fri, 14 Oct 2022 16:27:11 +0200 Subject: [PATCH 3/5] Restore 1.4.x behavior --- pandas/core/internals/blocks.py | 7 ++++--- pandas/tests/frame/methods/test_replace.py | 6 +++++- pandas/tests/series/methods/test_replace.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index c4fea3eec7520..9f38568c1bbb6 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -573,8 +573,6 @@ def replace( # Note: the checks we do in NDFrame.replace ensure we never get # here with listlike to_replace or value, as those cases # go through replace_list - value = self._standardize_fill_value(value) - values = self.values if isinstance(values, Categorical): @@ -613,7 +611,10 @@ def replace( return blocks elif self.ndim == 1 or self.shape[0] == 1: - blk = self.coerce_to_target_dtype(value) + if value is None: + blk = self.astype(object) + else: + blk = self.coerce_to_target_dtype(value) return blk.replace( to_replace=to_replace, value=value, diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index f41cc709c9bf2..f4de685688b00 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -1501,7 +1501,11 @@ def test_replace_value_none_dtype_numeric(self, val): # GH#48231 df = DataFrame({"a": [1, val]}) result = df.replace(val, None) - expected = DataFrame({"a": [1, np.nan]}) + expected = DataFrame({"a": [1, None]}, dtype=object) + tm.assert_frame_equal(result, expected) + + df = DataFrame({"a": [1, val]}) + result = df.replace({val: None}) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/methods/test_replace.py b/pandas/tests/series/methods/test_replace.py index 3129c5f405379..126a89503d636 100644 --- a/pandas/tests/series/methods/test_replace.py +++ b/pandas/tests/series/methods/test_replace.py @@ -673,5 +673,5 @@ def test_replace_value_none_dtype_numeric(self, val): # GH#48231 ser = pd.Series([1, val]) result = ser.replace(val, None) - expected = pd.Series([1, np.nan]) + expected = pd.Series([1, None], dtype=object) tm.assert_series_equal(result, expected) From a71367da030f3ad508c37138bf6c0ab736745759 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Fri, 14 Oct 2022 23:02:26 +0200 Subject: [PATCH 4/5] Fix mypy --- pandas/core/internals/blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 9f38568c1bbb6..e28f2f387fa5c 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -612,7 +612,7 @@ def replace( elif self.ndim == 1 or self.shape[0] == 1: if value is None: - blk = self.astype(object) + blk = self.astype(np.dtype(object)) else: blk = self.coerce_to_target_dtype(value) return blk.replace( From 077cd935563425ecc3c477669df87cb99e4203c5 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Sun, 23 Oct 2022 19:38:09 +0200 Subject: [PATCH 5/5] Move whatsnew --- doc/source/whatsnew/v1.5.1.rst | 1 - doc/source/whatsnew/v1.5.2.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v1.5.1.rst b/doc/source/whatsnew/v1.5.1.rst index 181334b2df420..2aa1562168fbb 100644 --- a/doc/source/whatsnew/v1.5.1.rst +++ b/doc/source/whatsnew/v1.5.1.rst @@ -73,7 +73,6 @@ Fixed regressions - Fixed Regression in :meth:`DataFrame.loc` when setting values as a :class:`DataFrame` with all ``True`` indexer (:issue:`48701`) - Regression in :func:`.read_csv` causing an ``EmptyDataError`` when using an UTF-8 file handle that was already read from (:issue:`48646`) - Regression in :func:`to_datetime` when ``utc=True`` and ``arg`` contained timezone naive and aware arguments raised a ``ValueError`` (:issue:`48678`) -- Fixed regression in :meth:`Series.replace` raising ``RecursionError`` with numeric dtype and when specifying ``value=None`` (:issue:`45725`) - Fixed regression in :meth:`DataFrame.loc` raising ``FutureWarning`` when setting an empty :class:`DataFrame` (:issue:`48480`) - Fixed regression in :meth:`DataFrame.describe` raising ``TypeError`` when result contains ``NA`` (:issue:`48778`) - Fixed regression in :meth:`DataFrame.plot` ignoring invalid ``colormap`` for ``kind="scatter"`` (:issue:`48726`) diff --git a/doc/source/whatsnew/v1.5.2.rst b/doc/source/whatsnew/v1.5.2.rst index aaf00804262bb..4f6274b9084da 100644 --- a/doc/source/whatsnew/v1.5.2.rst +++ b/doc/source/whatsnew/v1.5.2.rst @@ -13,7 +13,7 @@ including other versions of pandas. Fixed regressions ~~~~~~~~~~~~~~~~~ -- +- Fixed regression in :meth:`Series.replace` raising ``RecursionError`` with numeric dtype and when specifying ``value=None`` (:issue:`45725`) - .. ---------------------------------------------------------------------------