From cf5fc6d6a65ddde1480dfc636e0508c01309714d Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Fri, 18 Sep 2020 08:30:01 +0200 Subject: [PATCH 01/13] BUG: is_float added to IntBlock._can_hold_element Added return is_integer(element) or is_float(element) to the IntBlock._can_hold_element method because an Block of ints can be replaced from int. Read #35376 for more info --- pandas/core/internals/blocks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index eb5b887c8b0cb..e445cb6296810 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -38,6 +38,7 @@ is_extension_array_dtype, is_float_dtype, is_integer, + is_float, is_integer_dtype, is_interval_dtype, is_list_like, @@ -2066,7 +2067,7 @@ def _can_hold_element(self, element: Any) -> bool: and not issubclass(tipo.type, (np.datetime64, np.timedelta64)) and self.dtype.itemsize >= tipo.itemsize ) - return is_integer(element) + return is_integer(element) or is_float(element) class DatetimeLikeBlockMixin: From 3222d60ba4247e341df6950ca2d7ab3e7efa463d Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Fri, 18 Sep 2020 08:36:24 +0200 Subject: [PATCH 02/13] BUG: is_float(e) and e.is_integer() added to IntBlock._can_hold_element As @jbrockmendel said in #35376, you can replace an int by a float in an IntBlock only if the element is an integer. --- pandas/core/internals/blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index e445cb6296810..51e4ba9c36f34 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2067,7 +2067,7 @@ def _can_hold_element(self, element: Any) -> bool: and not issubclass(tipo.type, (np.datetime64, np.timedelta64)) and self.dtype.itemsize >= tipo.itemsize ) - return is_integer(element) or is_float(element) + return is_integer(element) or (is_float(element) and element.is_integer()) class DatetimeLikeBlockMixin: From 6a047cf36b562e79d995e6bcf603896e1759ba69 Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Fri, 18 Sep 2020 10:24:45 +0200 Subject: [PATCH 03/13] isort passed move an import 2 lines above --- pandas/core/internals/blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 51e4ba9c36f34..2321324dac429 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -36,9 +36,9 @@ is_datetime64tz_dtype, is_dtype_equal, is_extension_array_dtype, + is_float, is_float_dtype, is_integer, - is_float, is_integer_dtype, is_interval_dtype, is_list_like, From ff63b40e34f5978ec5144648d473967aff0199ca Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Fri, 18 Sep 2020 15:29:09 +0200 Subject: [PATCH 04/13] replace test added Added the #35376 error as a test. --- pandas/tests/frame/methods/test_replace.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index a77753ed9f9d0..73a43b2bfa0f1 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -974,6 +974,13 @@ def test_replace_for_new_dtypes(self, datetime_frame): } ), ), + # GH 35376 + ( + DataFrame([[1, 1.0], [2, 2.0]]), + 1.0, + 5, + DataFrame([[5, 5.0], [2, 2.0]]), + ), ], ) def test_replace_dtypes(self, frame, to_replace, value, expected): From 52ac6f50c9125d5ede4b43ce99c21089fe967d87 Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Fri, 18 Sep 2020 15:55:44 +0200 Subject: [PATCH 05/13] release note Added regression description in `doc/source/whatsnew/v1.1.3.rst` --- doc/source/whatsnew/v1.1.3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.3.rst b/doc/source/whatsnew/v1.1.3.rst index 3f8413bd492ca..ee2243c753be0 100644 --- a/doc/source/whatsnew/v1.1.3.rst +++ b/doc/source/whatsnew/v1.1.3.rst @@ -34,7 +34,7 @@ Fixed regressions - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a tuple (:issue:`35534`) - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a frozenset (:issue:`35747`) - Fixed regression in :meth:`read_excel` with ``engine="odf"`` caused ``UnboundLocalError`` in some cases where cells had nested child nodes (:issue:`36122`,:issue:`35802`) -- +- Fixed regression in :meth:`DataFrame.replace` inconsistant replace when using a float in the replace method (:issue:`35376`) .. --------------------------------------------------------------------------- From f64817b28e52a00e15da8beaae7f9ec01d7b8567 Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Fri, 18 Sep 2020 16:00:20 +0200 Subject: [PATCH 06/13] black on test_replace moved 4 lines in once --- pandas/tests/frame/methods/test_replace.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index 73a43b2bfa0f1..8adb247d613f2 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -975,12 +975,7 @@ def test_replace_for_new_dtypes(self, datetime_frame): ), ), # GH 35376 - ( - DataFrame([[1, 1.0], [2, 2.0]]), - 1.0, - 5, - DataFrame([[5, 5.0], [2, 2.0]]), - ), + (DataFrame([[1, 1.0], [2, 2.0]]), 1.0, 5, DataFrame([[5, 5.0], [2, 2.0]]),), ], ) def test_replace_dtypes(self, frame, to_replace, value, expected): From 23bd5ce973c4de45b903162d49f6e734c71c2592 Mon Sep 17 00:00:00 2001 From: Number42 <32516498+QuentinN42@users.noreply.github.com> Date: Fri, 18 Sep 2020 17:22:02 +0200 Subject: [PATCH 07/13] Update doc/source/whatsnew/v1.1.3.rst Thx @simonjayhawkins for the typo Co-authored-by: Simon Hawkins --- doc/source/whatsnew/v1.1.3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.3.rst b/doc/source/whatsnew/v1.1.3.rst index ee2243c753be0..c888ebf8f7cb7 100644 --- a/doc/source/whatsnew/v1.1.3.rst +++ b/doc/source/whatsnew/v1.1.3.rst @@ -34,7 +34,7 @@ Fixed regressions - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a tuple (:issue:`35534`) - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a frozenset (:issue:`35747`) - Fixed regression in :meth:`read_excel` with ``engine="odf"`` caused ``UnboundLocalError`` in some cases where cells had nested child nodes (:issue:`36122`,:issue:`35802`) -- Fixed regression in :meth:`DataFrame.replace` inconsistant replace when using a float in the replace method (:issue:`35376`) +- Fixed regression in :meth:`DataFrame.replace` inconsistent replace when using a float in the replace method (:issue:`35376`) .. --------------------------------------------------------------------------- From 473f1083b31ca69522ff45e521a151783a70f967 Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Mon, 21 Sep 2020 08:48:00 +0200 Subject: [PATCH 08/13] more tests on test_replace added .replace(1, 5), .replace(1.0, 5.0) and .replace(1, 5.0) --- pandas/tests/frame/methods/test_replace.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index 8adb247d613f2..dd4db9a21f27d 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -976,6 +976,9 @@ def test_replace_for_new_dtypes(self, datetime_frame): ), # GH 35376 (DataFrame([[1, 1.0], [2, 2.0]]), 1.0, 5, DataFrame([[5, 5.0], [2, 2.0]]),), + (DataFrame([[1, 1.0], [2, 2.0]]), 1, 5, DataFrame([[5, 5.0], [2, 2.0]]),), + (DataFrame([[1, 1.0], [2, 2.0]]), 1.0, 5.0, DataFrame([[5, 5.0], [2, 2.0]]),), + (DataFrame([[1, 1.0], [2, 2.0]]), 1, 5.0, DataFrame([[5, 5.0], [2, 2.0]]),), ], ) def test_replace_dtypes(self, frame, to_replace, value, expected): From 1355628dc5a6d4ea52bd3ec471e74b0a1019576d Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Mon, 21 Sep 2020 08:54:52 +0200 Subject: [PATCH 09/13] test_replace PEP8 Moved all singles lines in 6 lines --- pandas/tests/frame/methods/test_replace.py | 28 ++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index dd4db9a21f27d..a9cf840470ae0 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -975,10 +975,30 @@ def test_replace_for_new_dtypes(self, datetime_frame): ), ), # GH 35376 - (DataFrame([[1, 1.0], [2, 2.0]]), 1.0, 5, DataFrame([[5, 5.0], [2, 2.0]]),), - (DataFrame([[1, 1.0], [2, 2.0]]), 1, 5, DataFrame([[5, 5.0], [2, 2.0]]),), - (DataFrame([[1, 1.0], [2, 2.0]]), 1.0, 5.0, DataFrame([[5, 5.0], [2, 2.0]]),), - (DataFrame([[1, 1.0], [2, 2.0]]), 1, 5.0, DataFrame([[5, 5.0], [2, 2.0]]),), + ( + DataFrame([[1, 1.0], [2, 2.0]]), + 1.0, + 5, + DataFrame([[5, 5.0], [2, 2.0]]), + ), + ( + DataFrame([[1, 1.0], [2, 2.0]]), + 1, + 5, + DataFrame([[5, 5.0], [2, 2.0]]), + ), + ( + DataFrame([[1, 1.0], [2, 2.0]]), + 1.0, + 5.0, + DataFrame([[5, 5.0], [2, 2.0]]), + ), + ( + DataFrame([[1, 1.0], [2, 2.0]]), + 1, + 5.0, + DataFrame([[5, 5.0], [2, 2.0]]), + ), ], ) def test_replace_dtypes(self, frame, to_replace, value, expected): From 8b86da4920eee909ca4da40bb7b3698c0c539635 Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Mon, 21 Sep 2020 09:12:29 +0200 Subject: [PATCH 10/13] black on test_replace Formatting is just non sense --- pandas/tests/frame/methods/test_replace.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index a9cf840470ae0..1a39c22a2384a 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -975,30 +975,15 @@ def test_replace_for_new_dtypes(self, datetime_frame): ), ), # GH 35376 + (DataFrame([[1, 1.0], [2, 2.0]]), 1.0, 5, DataFrame([[5, 5.0], [2, 2.0]]),), + (DataFrame([[1, 1.0], [2, 2.0]]), 1, 5, DataFrame([[5, 5.0], [2, 2.0]]),), ( DataFrame([[1, 1.0], [2, 2.0]]), 1.0, - 5, - DataFrame([[5, 5.0], [2, 2.0]]), - ), - ( - DataFrame([[1, 1.0], [2, 2.0]]), - 1, - 5, - DataFrame([[5, 5.0], [2, 2.0]]), - ), - ( - DataFrame([[1, 1.0], [2, 2.0]]), - 1.0, - 5.0, - DataFrame([[5, 5.0], [2, 2.0]]), - ), - ( - DataFrame([[1, 1.0], [2, 2.0]]), - 1, 5.0, DataFrame([[5, 5.0], [2, 2.0]]), ), + (DataFrame([[1, 1.0], [2, 2.0]]), 1, 5.0, DataFrame([[5, 5.0], [2, 2.0]]),), ], ) def test_replace_dtypes(self, frame, to_replace, value, expected): From b86768131ad09e91259e545e792f59a23ce07750 Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Wed, 23 Sep 2020 09:15:03 +0200 Subject: [PATCH 11/13] added comment in IntBlock._can_hold_element marking GH 36444 and GH 35376 close to (is_float(element) and element.is_integer()) --- pandas/core/internals/blocks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 2321324dac429..fdbfaa54687f3 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2067,6 +2067,7 @@ def _can_hold_element(self, element: Any) -> bool: and not issubclass(tipo.type, (np.datetime64, np.timedelta64)) and self.dtype.itemsize >= tipo.itemsize ) + # Will have to be modified in the future see GH 36444 and GH 35376 return is_integer(element) or (is_float(element) and element.is_integer()) From 417bbdb658c942958ae6367e0d0db5db4b50e6d9 Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Wed, 23 Sep 2020 11:23:40 +0200 Subject: [PATCH 12/13] black test_replace.py refactor --- pandas/tests/frame/methods/test_replace.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pandas/tests/frame/methods/test_replace.py b/pandas/tests/frame/methods/test_replace.py index 1a39c22a2384a..a9cf840470ae0 100644 --- a/pandas/tests/frame/methods/test_replace.py +++ b/pandas/tests/frame/methods/test_replace.py @@ -975,15 +975,30 @@ def test_replace_for_new_dtypes(self, datetime_frame): ), ), # GH 35376 - (DataFrame([[1, 1.0], [2, 2.0]]), 1.0, 5, DataFrame([[5, 5.0], [2, 2.0]]),), - (DataFrame([[1, 1.0], [2, 2.0]]), 1, 5, DataFrame([[5, 5.0], [2, 2.0]]),), ( DataFrame([[1, 1.0], [2, 2.0]]), 1.0, + 5, + DataFrame([[5, 5.0], [2, 2.0]]), + ), + ( + DataFrame([[1, 1.0], [2, 2.0]]), + 1, + 5, + DataFrame([[5, 5.0], [2, 2.0]]), + ), + ( + DataFrame([[1, 1.0], [2, 2.0]]), + 1.0, + 5.0, + DataFrame([[5, 5.0], [2, 2.0]]), + ), + ( + DataFrame([[1, 1.0], [2, 2.0]]), + 1, 5.0, DataFrame([[5, 5.0], [2, 2.0]]), ), - (DataFrame([[1, 1.0], [2, 2.0]]), 1, 5.0, DataFrame([[5, 5.0], [2, 2.0]]),), ], ) def test_replace_dtypes(self, frame, to_replace, value, expected): From ada13b64f5ed73cf7db26a37fb7d3f9b6c371235 Mon Sep 17 00:00:00 2001 From: QuentinN42 Date: Thu, 24 Sep 2020 11:17:17 +0200 Subject: [PATCH 13/13] changing comment to be more explicit --- pandas/core/internals/blocks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index fdbfaa54687f3..644ed12655c9c 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2067,7 +2067,8 @@ def _can_hold_element(self, element: Any) -> bool: and not issubclass(tipo.type, (np.datetime64, np.timedelta64)) and self.dtype.itemsize >= tipo.itemsize ) - # Will have to be modified in the future see GH 36444 and GH 35376 + # We have not inferred an integer from the dtype + # check if we have a builtin int or a float equal to an int return is_integer(element) or (is_float(element) and element.is_integer())