From 3cf94ff97e83718b71608100633ca0b53c3455ae Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Fri, 24 Sep 2021 21:44:19 -0400 Subject: [PATCH 1/2] DEPR: Silent dropping of nuisance columns in transform_dict_like --- doc/source/whatsnew/v1.4.0.rst | 1 + pandas/core/apply.py | 13 ++++++------ pandas/tests/apply/test_frame_transform.py | 21 ++++++++++++------- pandas/tests/apply/test_series_apply.py | 24 +++++++++++++++------- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index 51c6b297a37cb..a7b4126a96030 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -337,6 +337,7 @@ Other Deprecations - Deprecated the ``squeeze`` argument to :meth:`read_csv`, :meth:`read_table`, and :meth:`read_excel`. Users should squeeze the DataFrame afterwards with ``.squeeze("columns")`` instead. (:issue:`43242`) - Deprecated the ``index`` argument to :class:`SparseArray` construction (:issue:`23089`) - Deprecated :meth:`.Rolling.validate`, :meth:`.Expanding.validate`, and :meth:`.ExponentialMovingWindow.validate` (:issue:`43665`) +- Deprecated silent dropping of columns that raised a ``TypeError`` in :class:`Series.transform` and :class:`DataFrame.transform` when used with a dictionary (:issue:``) .. --------------------------------------------------------------------------- diff --git a/pandas/core/apply.py b/pandas/core/apply.py index dafcc71ae0b5a..d23a512964ec0 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -33,6 +33,7 @@ FrameOrSeries, ) from pandas.util._decorators import cache_readonly +from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.cast import is_nested_object from pandas.core.dtypes.common import ( @@ -271,8 +272,9 @@ def transform_dict_like(self, func): "No transform functions were provided", }: raise err - elif not isinstance(err, TypeError): - all_type_errors = False + else: + if not isinstance(err, TypeError): + all_type_errors = False failed_names.append(name) # combine results if not results: @@ -280,12 +282,11 @@ def transform_dict_like(self, func): raise klass("Transform function failed") if len(failed_names) > 0: warnings.warn( - f"{failed_names} did not transform successfully and did not raise " - f"a TypeError. If any error is raised except for TypeError, " - f"this will raise in a future version of pandas. " + f"{failed_names} did not transform successfully. If any error is " + f"raised, this will raise in a future version of pandas. " f"Drop these columns/ops to avoid this warning.", FutureWarning, - stacklevel=4, + stacklevel=find_stack_level(), ) return concat(results, axis=1) diff --git a/pandas/tests/apply/test_frame_transform.py b/pandas/tests/apply/test_frame_transform.py index 47173d14c543d..ade8df6fbfc0e 100644 --- a/pandas/tests/apply/test_frame_transform.py +++ b/pandas/tests/apply/test_frame_transform.py @@ -160,7 +160,6 @@ def test_transform_bad_dtype(op, frame_or_series, request): @pytest.mark.parametrize("op", frame_kernels_raise) def test_transform_partial_failure_typeerror(op): # GH 35964 - if op == "rank": pytest.skip("GH 40418: rank does not raise a TypeError") @@ -168,25 +167,33 @@ def test_transform_partial_failure_typeerror(op): df = DataFrame({"A": 3 * [object], "B": [1, 2, 3]}) expected = df[["B"]].transform([op]) - result = df.transform([op]) + match = r"\['A'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + result = df.transform([op]) tm.assert_equal(result, expected) expected = df[["B"]].transform({"B": op}) - result = df.transform({"A": op, "B": op}) + match = r"\['A'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + result = df.transform({"A": op, "B": op}) tm.assert_equal(result, expected) expected = df[["B"]].transform({"B": [op]}) - result = df.transform({"A": [op], "B": [op]}) + match = r"\['A'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + result = df.transform({"A": [op], "B": [op]}) tm.assert_equal(result, expected) expected = df.transform({"A": ["shift"], "B": [op]}) - result = df.transform({"A": [op, "shift"], "B": [op]}) + match = rf"\['{op}'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + result = df.transform({"A": [op, "shift"], "B": [op]}) tm.assert_equal(result, expected) def test_transform_partial_failure_valueerror(): # GH 40211 - match = ".*did not transform successfully and did not raise a TypeError" + match = ".*did not transform successfully" def op(x): if np.sum(np.sum(x)) < 10: @@ -211,7 +218,7 @@ def op(x): tm.assert_equal(result, expected) expected = df.transform({"A": ["shift"], "B": [op]}) - with tm.assert_produces_warning(FutureWarning, match=match, check_stacklevel=False): + with tm.assert_produces_warning(FutureWarning, match=match): result = df.transform({"A": [op, "shift"], "B": [op]}) tm.assert_equal(result, expected) diff --git a/pandas/tests/apply/test_series_apply.py b/pandas/tests/apply/test_series_apply.py index 2d0202fbeb6c8..18f96f9c61ab8 100644 --- a/pandas/tests/apply/test_series_apply.py +++ b/pandas/tests/apply/test_series_apply.py @@ -291,25 +291,35 @@ def test_transform_partial_failure(op, request): ser = Series(3 * [object]) expected = ser.transform(["shift"]) - result = ser.transform([op, "shift"]) + match = rf"\['{op}'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + result = ser.transform([op, "shift"]) tm.assert_equal(result, expected) expected = ser.transform({"B": "shift"}) - result = ser.transform({"A": op, "B": "shift"}) + match = r"\['A'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + result = ser.transform({"A": op, "B": "shift"}) tm.assert_equal(result, expected) expected = ser.transform({"B": ["shift"]}) - result = ser.transform({"A": [op], "B": ["shift"]}) + match = r"\['A'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + result = ser.transform({"A": [op], "B": ["shift"]}) tm.assert_equal(result, expected) - expected = ser.transform({"A": ["shift"], "B": [op]}) - result = ser.transform({"A": [op, "shift"], "B": [op]}) + match = r"\['B'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + expected = ser.transform({"A": ["shift"], "B": [op]}) + match = rf"\['{op}'\] did not transform successfully" + with tm.assert_produces_warning(FutureWarning, match=match): + result = ser.transform({"A": [op, "shift"], "B": [op]}) tm.assert_equal(result, expected) def test_transform_partial_failure_valueerror(): # GH 40211 - match = ".*did not transform successfully and did not raise a TypeError" + match = ".*did not transform successfully" def noop(x): return x @@ -335,7 +345,7 @@ def raising_op(_): tm.assert_equal(result, expected) expected = ser.transform({"A": [noop], "B": [noop]}) - with tm.assert_produces_warning(FutureWarning, match=match, check_stacklevel=False): + with tm.assert_produces_warning(FutureWarning, match=match): result = ser.transform({"A": [noop, raising_op], "B": [noop]}) tm.assert_equal(result, expected) From e028699c83e5332ed318a34cbf9df6f5833fa087 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 25 Sep 2021 13:21:15 -0400 Subject: [PATCH 2/2] Issue # for whatsnew --- doc/source/whatsnew/v1.4.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index a7b4126a96030..243bcf6900d2e 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -337,7 +337,7 @@ Other Deprecations - Deprecated the ``squeeze`` argument to :meth:`read_csv`, :meth:`read_table`, and :meth:`read_excel`. Users should squeeze the DataFrame afterwards with ``.squeeze("columns")`` instead. (:issue:`43242`) - Deprecated the ``index`` argument to :class:`SparseArray` construction (:issue:`23089`) - Deprecated :meth:`.Rolling.validate`, :meth:`.Expanding.validate`, and :meth:`.ExponentialMovingWindow.validate` (:issue:`43665`) -- Deprecated silent dropping of columns that raised a ``TypeError`` in :class:`Series.transform` and :class:`DataFrame.transform` when used with a dictionary (:issue:``) +- Deprecated silent dropping of columns that raised a ``TypeError`` in :class:`Series.transform` and :class:`DataFrame.transform` when used with a dictionary (:issue:`43740`) .. ---------------------------------------------------------------------------