DEP: Partial failure in Series.transform and DataFrame.transform.

Author: rhshadrach

doc/source/whatsnew/v1.3.0.rst

@@ -363,6 +363,7 @@ Deprecations
 - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`)
 - Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`)
 - Deprecated casting ``datetime.date`` objects to ``datetime64`` when used as ``fill_value`` in :meth:`DataFrame.unstack`, :meth:`DataFrame.shift`, :meth:`Series.shift`, and :meth:`DataFrame.reindex`, pass ``pd.Timestamp(dateobj)`` instead (:issue:`39767`)
+- Deprecated allowing partial failure in :meth:`Series.transform` and :meth:`DataFrame.transform` when ``func`` is list-like and dict-like; will raise if any function fails on column in a future version (:issue:`40211`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/apply.py

@@ -15,6 +15,7 @@
     Union,
     cast,
 )
+import warnings
 
 import numpy as np
 
@@ -267,6 +268,7 @@ def transform_dict_like(self, func):
         func = self.normalize_dictlike_arg("transform", obj, func)
 
         results: Dict[Hashable, FrameOrSeriesUnion] = {}
+        failed_names = []
         for name, how in func.items():
             colg = obj._gotitem(name, ndim=1)
             try:
@@ -277,10 +279,21 @@ def transform_dict_like(self, func):
                     "No transform functions were provided",
                 }:
                     raise err
-
+                else:
+                    failed_names.append(name)
         # combine results
         if not results:
             raise ValueError("Transform function failed")
+        if len(failed_names) > 0:
+            warnings.warn(
+                f"Columns {failed_names} did not transform successfully. "
+                f"Allowing for partial failure is deprecated, this will raise "
+                f"a ValueError in a future version of pandas."
+                f"Drop these columns before calling transform to avoid"
+                f"this warning.",
+                FutureWarning,
+                stacklevel=4,
+            )
         return concat(results, axis=1)
 
     def transform_str_or_callable(self, func) -> FrameOrSeriesUnion:

pandas/tests/apply/common.py

@@ -0,0 +1,10 @@
+from pandas.core.groupby.base import transformation_kernels
+
+# tshift only works on time index and is deprecated
+# There is no Series.cumcount or DataFrame.cumcount
+series_transform_kernels = [
+    x for x in sorted(transformation_kernels) if x not in ["tshift", "cumcount"]
+]
+frame_transform_kernels = [
+    x for x in sorted(transformation_kernels) if x not in ["tshift", "cumcount"]
+]

pandas/tests/apply/test_frame_transform.py

@@ -9,15 +9,10 @@
     Series,
 )
 import pandas._testing as tm
-from pandas.core.groupby.base import transformation_kernels
+from pandas.core.base import SpecificationError
+from pandas.tests.apply.common import frame_transform_kernels
 from pandas.tests.frame.common import zip_frames
 
-# tshift only works on time index and is deprecated
-# There is no DataFrame.cumcount
-frame_kernels = [
-    x for x in sorted(transformation_kernels) if x not in ["tshift", "cumcount"]
-]
-
 
 def unpack_obj(obj, klass, axis):
     """
@@ -44,7 +39,7 @@ def test_transform_ufunc(axis, float_frame, frame_or_series):
     tm.assert_equal(result, expected)
 
 
-@pytest.mark.parametrize("op", frame_kernels)
+@pytest.mark.parametrize("op", frame_transform_kernels)
 def test_transform_groupby_kernel(axis, float_frame, op):
     # GH 35964
 
@@ -158,7 +153,7 @@ def test_transform_method_name(method):
 
 
 wont_fail = ["ffill", "bfill", "fillna", "pad", "backfill", "shift"]
-frame_kernels_raise = [x for x in frame_kernels if x not in wont_fail]
+frame_kernels_raise = [x for x in frame_transform_kernels if x not in wont_fail]
 
 
 # mypy doesn't allow adding lists of different types
@@ -187,21 +182,25 @@ def test_transform_bad_dtype(op, frame_or_series):
 
 @pytest.mark.parametrize("op", frame_kernels_raise)
 def test_transform_partial_failure(op):
-    # GH 35964
+    # GH 35964 & GH 40211
+    match = "Allowing for partial failure is deprecated"
 
     # Using object makes most transform kernels fail
     df = DataFrame({"A": 3 * [object], "B": [1, 2, 3]})
 
     expected = df[["B"]].transform([op])
-    result = df.transform([op])
+    with tm.assert_produces_warning(FutureWarning, match=match):
+        result = df.transform([op])
     tm.assert_equal(result, expected)
 
     expected = df[["B"]].transform({"B": op})
-    result = df.transform({"B": op})
+    with tm.assert_produces_warning(FutureWarning, match=match):
+        result = df.transform({"A": op, "B": op})
     tm.assert_equal(result, expected)
 
     expected = df[["B"]].transform({"B": [op]})
-    result = df.transform({"B": [op]})
+    with tm.assert_produces_warning(FutureWarning, match=match):
+        result = df.transform({"A": [op], "B": [op]})
     tm.assert_equal(result, expected)
 
 

pandas/tests/apply/test_series_apply.py

@@ -20,6 +20,8 @@
     timedelta_range,
 )
 import pandas._testing as tm
+from pandas.core.base import SpecificationError
+from pandas.tests.apply.common import series_transform_kernels
 
 
 def test_series_map_box_timedelta():
@@ -256,6 +258,47 @@ def test_transform(string_series):
         tm.assert_series_equal(result.reindex_like(expected), expected)
 
 
+@pytest.mark.parametrize("op", series_transform_kernels)
+def test_transform_partial_failure(op, request):
+    # GH 35964 & GH 40211
+    if op in ("ffill", "bfill", "pad", "backfill", "shift"):
+        request.node.add_marker(
+            pytest.mark.xfail(reason=f"{op} is successful on any dtype")
+        )
+    match = "Allowing for partial failure is deprecated"
+
+    # Using object makes most transform kernels fail
+    ser = Series(3 * [object])
+
+    expected = ser.transform(["shift"])
+    with tm.assert_produces_warning(FutureWarning, match=match):
+        result = ser.transform([op, "shift"])
+    tm.assert_equal(result, expected)
+
+    expected = ser.transform({"B": "shift"})
+    with tm.assert_produces_warning(FutureWarning, match=match):
+        result = ser.transform({"A": op, "B": "shift"})
+    tm.assert_equal(result, expected)
+
+    expected = ser.transform({"B": ["shift"]})
+    with tm.assert_produces_warning(FutureWarning, match=match):
+        result = ser.transform({"A": [op], "B": ["shift"]})
+    tm.assert_equal(result, expected)
+
+
+def test_transform_and_agg_error(string_series):
+    # we are trying to transform with an aggregator
+    msg = "cannot combine transform and aggregation"
+    with pytest.raises(ValueError, match=msg):
+        with np.errstate(all="ignore"):
+            string_series.agg(["sqrt", "max"])
+
+    msg = "cannot perform both aggregation and transformation"
+    with pytest.raises(ValueError, match=msg):
+        with np.errstate(all="ignore"):
+            string_series.agg({"foo": np.sqrt, "bar": "sum"})
+
+
 def test_demo():
     # demonstration tests
     s = Series(range(6), dtype="int64", name="series")