From 138d9d27ff7ac30559ae0381a2ac2fc0afec051a Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 09:21:36 -0400 Subject: [PATCH 01/12] REGR: ufunc args not being passed --- pandas/core/arraylike.py | 2 +- pandas/tests/frame/test_ufunc.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pandas/core/arraylike.py b/pandas/core/arraylike.py index b110d62b606d9..620c1de693edf 100644 --- a/pandas/core/arraylike.py +++ b/pandas/core/arraylike.py @@ -357,7 +357,7 @@ def reconstruct(result): # * len(inputs) > 1 is doable when we know that we have # aligned blocks / dtypes. inputs = tuple(np.asarray(x) for x in inputs) - result = getattr(ufunc, method)(*inputs) + result = getattr(ufunc, method)(*inputs, **kwargs) elif self.ndim == 1: # ufunc(series, ...) inputs = tuple(extract_array(x, extract_numpy=True) for x in inputs) diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index 0bcb62251fc5c..cffc5c0005349 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -60,6 +60,16 @@ def test_binary_input_dispatch_binop(dtype): tm.assert_frame_equal(result, expected) +def test_ufunc_passes_args(): + # GH#40662 + arr = np.array([[1, 2], [3, 4]]) + df = pd.DataFrame(arr) + arr_to_modify = np.zeros_like(df) + np.add(df, 1, out=arr_to_modify) + + tm.assert_numpy_array_equal(arr + 1, arr_to_modify) + + @pytest.mark.parametrize("dtype_a", dtypes) @pytest.mark.parametrize("dtype_b", dtypes) def test_binary_input_aligns_columns(request, dtype_a, dtype_b): From 3b492e416275ebb52d6187242e134dc094115112 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 09:29:04 -0400 Subject: [PATCH 02/12] Add whatsnew --- doc/source/whatsnew/v1.2.4.rst | 1 + pandas/tests/frame/test_ufunc.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v1.2.4.rst b/doc/source/whatsnew/v1.2.4.rst index fffdf333178fc..9b6bc491b65d1 100644 --- a/doc/source/whatsnew/v1.2.4.rst +++ b/doc/source/whatsnew/v1.2.4.rst @@ -21,6 +21,7 @@ Fixed regressions - Fixed regression in :meth:`DataFrame.where` not returning a copy in the case of an all True condition (:issue:`39595`) - Fixed regression in :meth:`DataFrame.replace` raising ``IndexError`` when ``regex`` was a multi-key dictionary (:issue:`39338`) - Fixed regression in repr of floats in an ``object`` column not respecting ``float_format`` when printed in the console or outputted through :meth:`DataFrame.to_string`, :meth:`DataFrame.to_html`, and :meth:`DataFrame.to_latex` (:issue:`40024`) +- Fixed regression in ``numpy`` ufunc such as ``np.add`` not passing through all arguments for 2-dimensional input (:issue:`40662`) .. --------------------------------------------------------------------------- diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index cffc5c0005349..ec938954cdc34 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -64,10 +64,16 @@ def test_ufunc_passes_args(): # GH#40662 arr = np.array([[1, 2], [3, 4]]) df = pd.DataFrame(arr) - arr_to_modify = np.zeros_like(df) - np.add(df, 1, out=arr_to_modify) + result = np.zeros_like(df) + np.add(df, 1, out=result) - tm.assert_numpy_array_equal(arr + 1, arr_to_modify) + expected = arr + 1 + tm.assert_numpy_array_equal(result, expected) + + result = np.zeros_like(df) + np.add(df, 1, out=result, where=[[False, True], [True, False]]) + expected = np.array([[0, 3], [4, 0]]) + tm.assert_numpy_array_equal(result, expected) @pytest.mark.parametrize("dtype_a", dtypes) From 625ce368b46071c9cc2b158369b90d3334e89073 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 09:36:20 -0400 Subject: [PATCH 03/12] Fix typo --- doc/source/whatsnew/v1.2.4.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.4.rst b/doc/source/whatsnew/v1.2.4.rst index 9b6bc491b65d1..703470bbf443d 100644 --- a/doc/source/whatsnew/v1.2.4.rst +++ b/doc/source/whatsnew/v1.2.4.rst @@ -21,7 +21,7 @@ Fixed regressions - Fixed regression in :meth:`DataFrame.where` not returning a copy in the case of an all True condition (:issue:`39595`) - Fixed regression in :meth:`DataFrame.replace` raising ``IndexError`` when ``regex`` was a multi-key dictionary (:issue:`39338`) - Fixed regression in repr of floats in an ``object`` column not respecting ``float_format`` when printed in the console or outputted through :meth:`DataFrame.to_string`, :meth:`DataFrame.to_html`, and :meth:`DataFrame.to_latex` (:issue:`40024`) -- Fixed regression in ``numpy`` ufunc such as ``np.add`` not passing through all arguments for 2-dimensional input (:issue:`40662`) +- Fixed regression in ``numpy`` ufuncs such as ``np.add`` not passing through all arguments for 2-dimensional input (:issue:`40662`) .. --------------------------------------------------------------------------- From 6bf0da0423e40937c7049dcbc0f2edf41d3b3d7e Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 10:09:27 -0400 Subject: [PATCH 04/12] Change shape --- pandas/tests/frame/test_ufunc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index ec938954cdc34..e19b81d69e197 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -62,7 +62,7 @@ def test_binary_input_dispatch_binop(dtype): def test_ufunc_passes_args(): # GH#40662 - arr = np.array([[1, 2], [3, 4]]) + arr = np.array([[1, 2, 3, 4]]) df = pd.DataFrame(arr) result = np.zeros_like(df) np.add(df, 1, out=result) @@ -71,8 +71,8 @@ def test_ufunc_passes_args(): tm.assert_numpy_array_equal(result, expected) result = np.zeros_like(df) - np.add(df, 1, out=result, where=[[False, True], [True, False]]) - expected = np.array([[0, 3], [4, 0]]) + np.add(df, 1, out=result, where=[[False, True, True, False]]) + expected = np.array([[0, 3, 4, 0]]) tm.assert_numpy_array_equal(result, expected) From 7c9a63fa9bdffe3a4cbfe1cf0228a4f012cd10a4 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 10:18:05 -0400 Subject: [PATCH 05/12] Parameterize test --- pandas/tests/frame/test_ufunc.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index e19b81d69e197..2fa46ebb3059f 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -1,3 +1,5 @@ +from functools import partial + import numpy as np import pytest @@ -60,19 +62,22 @@ def test_binary_input_dispatch_binop(dtype): tm.assert_frame_equal(result, expected) -def test_ufunc_passes_args(): +@pytest.mark.parametrize( + "func,expected", + [ + (np.add, [2, 3, 4, 5]), + (partial(np.add, where=[[False, True], [True, False]]), [0, 3, 4, 0]), + (np.power, [1, 2, 3, 4]), + ], +) +def test_ufunc_passes_args(frame_or_series, func, expected): # GH#40662 - arr = np.array([[1, 2, 3, 4]]) + arr = np.array([[1, 2], [3, 4]]) df = pd.DataFrame(arr) result = np.zeros_like(df) - np.add(df, 1, out=result) - - expected = arr + 1 - tm.assert_numpy_array_equal(result, expected) + func(df, 1, out=result) - result = np.zeros_like(df) - np.add(df, 1, out=result, where=[[False, True, True, False]]) - expected = np.array([[0, 3, 4, 0]]) + expected = np.array(expected).reshape(2, 2) tm.assert_numpy_array_equal(result, expected) From cfb0bcd5d362d488b9c176704900159d039f6bf4 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 10:22:41 -0400 Subject: [PATCH 06/12] Remove unused --- pandas/tests/frame/test_ufunc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index 2fa46ebb3059f..04a33387a6e26 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -70,7 +70,7 @@ def test_binary_input_dispatch_binop(dtype): (np.power, [1, 2, 3, 4]), ], ) -def test_ufunc_passes_args(frame_or_series, func, expected): +def test_ufunc_passes_args(func, expected): # GH#40662 arr = np.array([[1, 2], [3, 4]]) df = pd.DataFrame(arr) From 58f73993e91a9c86e2256b86d82609208e3d3b30 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 10:28:30 -0400 Subject: [PATCH 07/12] Also test return value --- pandas/tests/frame/test_ufunc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index 04a33387a6e26..e0cbe47c023f1 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -74,11 +74,14 @@ def test_ufunc_passes_args(func, expected): # GH#40662 arr = np.array([[1, 2], [3, 4]]) df = pd.DataFrame(arr) - result = np.zeros_like(df) - func(df, 1, out=result) + result_inplace = np.zeros_like(df) + result = func(df, 1, out=result_inplace) expected = np.array(expected).reshape(2, 2) - tm.assert_numpy_array_equal(result, expected) + tm.assert_numpy_array_equal(result_inplace, expected) + + expected = pd.DataFrame(expected) + tm.assert_frame_equal(result, expected) @pytest.mark.parametrize("dtype_a", dtypes) From 459092684dc9afd24f5284949f9c5388ff4ec328 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin <37011898+mzeitlin11@users.noreply.github.com> Date: Sun, 11 Apr 2021 10:57:22 -0400 Subject: [PATCH 08/12] Update doc/source/whatsnew/v1.2.4.rst Co-authored-by: Simon Hawkins --- doc/source/whatsnew/v1.2.4.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.4.rst b/doc/source/whatsnew/v1.2.4.rst index 703470bbf443d..06a9d9d2365f9 100644 --- a/doc/source/whatsnew/v1.2.4.rst +++ b/doc/source/whatsnew/v1.2.4.rst @@ -21,7 +21,7 @@ Fixed regressions - Fixed regression in :meth:`DataFrame.where` not returning a copy in the case of an all True condition (:issue:`39595`) - Fixed regression in :meth:`DataFrame.replace` raising ``IndexError`` when ``regex`` was a multi-key dictionary (:issue:`39338`) - Fixed regression in repr of floats in an ``object`` column not respecting ``float_format`` when printed in the console or outputted through :meth:`DataFrame.to_string`, :meth:`DataFrame.to_html`, and :meth:`DataFrame.to_latex` (:issue:`40024`) -- Fixed regression in ``numpy`` ufuncs such as ``np.add`` not passing through all arguments for 2-dimensional input (:issue:`40662`) +- Fixed regression in NumPy ufuncs such as ``np.add`` not passing through all arguments for 2-dimensional input (:issue:`40662`) .. --------------------------------------------------------------------------- From d58677ebef2bfd4ffe67733d7f10dea6334e370c Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin <37011898+mzeitlin11@users.noreply.github.com> Date: Sun, 11 Apr 2021 10:59:39 -0400 Subject: [PATCH 09/12] Update doc/source/whatsnew/v1.2.4.rst --- doc/source/whatsnew/v1.2.4.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.4.rst b/doc/source/whatsnew/v1.2.4.rst index 06a9d9d2365f9..f9d62ba9bd61c 100644 --- a/doc/source/whatsnew/v1.2.4.rst +++ b/doc/source/whatsnew/v1.2.4.rst @@ -21,7 +21,7 @@ Fixed regressions - Fixed regression in :meth:`DataFrame.where` not returning a copy in the case of an all True condition (:issue:`39595`) - Fixed regression in :meth:`DataFrame.replace` raising ``IndexError`` when ``regex`` was a multi-key dictionary (:issue:`39338`) - Fixed regression in repr of floats in an ``object`` column not respecting ``float_format`` when printed in the console or outputted through :meth:`DataFrame.to_string`, :meth:`DataFrame.to_html`, and :meth:`DataFrame.to_latex` (:issue:`40024`) -- Fixed regression in NumPy ufuncs such as ``np.add`` not passing through all arguments for 2-dimensional input (:issue:`40662`) +- Fixed regression in NumPy ufuncs such as ``np.add`` not passing through all arguments for :class:`DataFrame` (:issue:`40662`) .. --------------------------------------------------------------------------- From 9c0b96c757e7255baa0968f32621341ae0d97287 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 11:11:16 -0400 Subject: [PATCH 10/12] Add subtract also --- pandas/tests/frame/test_ufunc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index e0cbe47c023f1..236c8959ed7fc 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -68,6 +68,7 @@ def test_binary_input_dispatch_binop(dtype): (np.add, [2, 3, 4, 5]), (partial(np.add, where=[[False, True], [True, False]]), [0, 3, 4, 0]), (np.power, [1, 2, 3, 4]), + (np.subtract, [0, 1, 2, 3]), ], ) def test_ufunc_passes_args(func, expected): From a2da4fc1f34bda32a3aa30529ccfd0f4e7dbc101 Mon Sep 17 00:00:00 2001 From: Matthew Zeitlin Date: Sun, 11 Apr 2021 15:22:33 -0400 Subject: [PATCH 11/12] Add test for other path --- pandas/core/arraylike.py | 2 +- pandas/tests/frame/test_ufunc.py | 33 ++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/pandas/core/arraylike.py b/pandas/core/arraylike.py index 620c1de693edf..267f49acd252c 100644 --- a/pandas/core/arraylike.py +++ b/pandas/core/arraylike.py @@ -367,7 +367,7 @@ def reconstruct(result): if method == "__call__": # for np.(..) calls mgr = inputs[0]._mgr - result = mgr.apply(getattr(ufunc, method)) + result = mgr.apply(getattr(ufunc, method), **kwargs) else: # otherwise specific ufunc methods (eg np..accumulate(..)) # Those can have an axis keyword and thus can't be called block-by-block diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index 236c8959ed7fc..5d7cd88f8be19 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -63,20 +63,37 @@ def test_binary_input_dispatch_binop(dtype): @pytest.mark.parametrize( - "func,expected", + "func,arg,expected", [ - (np.add, [2, 3, 4, 5]), - (partial(np.add, where=[[False, True], [True, False]]), [0, 3, 4, 0]), - (np.power, [1, 2, 3, 4]), - (np.subtract, [0, 1, 2, 3]), + (np.add, 1, [2, 3, 4, 5]), + ( + partial(np.add, where=[[False, True], [True, False]]), + np.array([[1, 1], [1, 1]]), + [0, 3, 4, 0], + ), + (np.power, np.array([[1, 1], [2, 2]]), [1, 2, 9, 16]), + (np.subtract, 2, [-1, 0, 1, 2]), + ( + partial(np.negative, where=np.array([[False, True], [True, False]])), + None, + [0, -2, -3, 0], + ), ], ) -def test_ufunc_passes_args(func, expected): +def test_ufunc_passes_args(func, arg, expected, request): # GH#40662 arr = np.array([[1, 2], [3, 4]]) df = pd.DataFrame(arr) - result_inplace = np.zeros_like(df) - result = func(df, 1, out=result_inplace) + result_inplace = np.zeros_like(arr) + # 1-argument ufunc + if arg is None: + mark = pytest.mark.xfail( + reason="Result becomes transposed with block-wise apply of ufunc, #40878" + ) + request.node.add_marker(mark) + result = func(df, out=result_inplace) + else: + result = func(df, arg, out=result_inplace) expected = np.array(expected).reshape(2, 2) tm.assert_numpy_array_equal(result_inplace, expected) From 5c5772774e88cf81fc3b572ed8e69b04358d0f2f Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 12 Apr 2021 14:35:55 +0200 Subject: [PATCH 12/12] check kwargs before calling mgr.apply --- pandas/core/arraylike.py | 6 ++++-- pandas/tests/frame/test_ufunc.py | 4 ---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pandas/core/arraylike.py b/pandas/core/arraylike.py index 267f49acd252c..031ff6a7665d4 100644 --- a/pandas/core/arraylike.py +++ b/pandas/core/arraylike.py @@ -364,10 +364,12 @@ def reconstruct(result): result = getattr(ufunc, method)(*inputs, **kwargs) else: # ufunc(dataframe) - if method == "__call__": + if method == "__call__" and not kwargs: # for np.(..) calls + # kwargs cannot necessarily be handled block-by-block, so only + # take this path if there are no kwargs mgr = inputs[0]._mgr - result = mgr.apply(getattr(ufunc, method), **kwargs) + result = mgr.apply(getattr(ufunc, method)) else: # otherwise specific ufunc methods (eg np..accumulate(..)) # Those can have an axis keyword and thus can't be called block-by-block diff --git a/pandas/tests/frame/test_ufunc.py b/pandas/tests/frame/test_ufunc.py index 5d7cd88f8be19..bdc4694d21963 100644 --- a/pandas/tests/frame/test_ufunc.py +++ b/pandas/tests/frame/test_ufunc.py @@ -87,10 +87,6 @@ def test_ufunc_passes_args(func, arg, expected, request): result_inplace = np.zeros_like(arr) # 1-argument ufunc if arg is None: - mark = pytest.mark.xfail( - reason="Result becomes transposed with block-wise apply of ufunc, #40878" - ) - request.node.add_marker(mark) result = func(df, out=result_inplace) else: result = func(df, arg, out=result_inplace)