From 3384b83768ad3c394b8b5a63e5cf62d64da7078d Mon Sep 17 00:00:00 2001 From: Matias Lindgren Date: Sun, 14 Jul 2024 11:46:13 +0200 Subject: [PATCH 1/8] throw when frame apply is given invalid axis+func --- pandas/core/apply.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 607a65598783f..c0b90d3148425 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -90,16 +90,20 @@ def frame_apply( kwargs=None, ) -> FrameApply: """construct and return a row or column based frame apply object""" + _, func, columns, _ = reconstruct_func(func, **kwargs) + axis = obj._get_axis_number(axis) klass: type[FrameApply] if axis == 0: klass = FrameRowApply elif axis == 1: + if columns: + raise NotImplementedError( + "func given to frame_apply cannot contain " + "an index relabeling when axis is 1" + ) klass = FrameColumnApply - _, func, _, _ = reconstruct_func(func, **kwargs) - assert func is not None - return klass( obj, func, From 079cde5b25a2ab87c285314fa537ec8c9dd69611 Mon Sep 17 00:00:00 2001 From: Matias Lindgren Date: Sun, 14 Jul 2024 11:46:35 +0200 Subject: [PATCH 2/8] test that frame_apply throws on invalid func+axis --- pandas/tests/apply/test_frame_apply.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/tests/apply/test_frame_apply.py b/pandas/tests/apply/test_frame_apply.py index 939997f44c1a9..4e7a2eecfcda2 100644 --- a/pandas/tests/apply/test_frame_apply.py +++ b/pandas/tests/apply/test_frame_apply.py @@ -1329,6 +1329,10 @@ def test_agg_reduce(axis, float_frame): expected = expected.T if axis in {1, "columns"} else expected tm.assert_frame_equal(result, expected) + msg = "func given to frame_apply cannot contain an index relabeling when axis is 1" + with pytest.raises(NotImplementedError, match=msg): + float_frame.agg(row1=(name1, "sum"), row2=(name2, "max"), axis=1) + def test_nuiscance_columns(): # GH 15015 From 8132f7e22ed2d374a048fa784d540cdd95e25f6b Mon Sep 17 00:00:00 2001 From: Matias Lindgren Date: Sun, 14 Jul 2024 11:47:13 +0200 Subject: [PATCH 3/8] add a note on unsupported func+axis combination for frame_apply --- pandas/core/shared_docs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/shared_docs.py b/pandas/core/shared_docs.py index 38a443b56ee3d..15c3df683f611 100644 --- a/pandas/core/shared_docs.py +++ b/pandas/core/shared_docs.py @@ -49,6 +49,8 @@ for more details. A passed user-defined-function will be passed a Series for evaluation. + +If `func` defines an index relabeling, `axis` must be `0` or `index`. {examples}""" _shared_docs["compare"] = """ From bd1ce77bb8460f34a3675a789dd474518a78637a Mon Sep 17 00:00:00 2001 From: Matias Lindgren Date: Sun, 14 Jul 2024 12:10:18 +0200 Subject: [PATCH 4/8] add bug fix to release notes --- doc/source/whatsnew/v3.0.0.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index ef06f57f611d1..531a0c601fb54 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -116,9 +116,18 @@ These improvements also fixed certain bugs in groupby: - :meth:`.DataFrameGroupBy.sum` would have incorrect values when there are multiple groupings, unobserved groups, and non-numeric data (:issue:`43891`) - :meth:`.DataFrameGroupBy.value_counts` would produce incorrect results when used with some categorical and some non-categorical groupings and ``observed=False`` (:issue:`56016`) -.. _whatsnew_300.notable_bug_fixes.notable_bug_fix2: +.. _whatsnew_300.notable_bug_fixes.improved_error_message_agg: -notable_bug_fix2 +Improved error message for :meth:`DataFrame.agg` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When :meth:`DataFrame.agg` is called with ``axis=1`` and a ``func`` which relabels the result index, the :func:`relabel_result` function iterates over result columns rather than the result rows, causing a confusing chain of ``KeyError`` exceptions (:issue:`58807`). + +This error is now handled in :func:`frame_apply` by throwing a ``NotImplementedError`` with a more explicit error message. + +.. _whatsnew_300.notable_bug_fixes.notable_bug_fix3: + +notable_bug_fix3 ^^^^^^^^^^^^^^^^ .. --------------------------------------------------------------------------- From 26e0f9a180587c5a541f75bba2667380210041a1 Mon Sep 17 00:00:00 2001 From: Matias Lindgren Date: Mon, 15 Jul 2024 21:18:32 +0200 Subject: [PATCH 5/8] fix based on review comments --- doc/source/whatsnew/v3.0.0.rst | 14 +++----------- pandas/core/apply.py | 3 +-- pandas/core/shared_docs.py | 2 +- pandas/tests/apply/test_frame_apply.py | 8 ++++++-- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 531a0c601fb54..6b7f9659d7449 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -28,6 +28,7 @@ enhancement2 Other enhancements ^^^^^^^^^^^^^^^^^^ +- Fix confusing chain of ``KeyError`` exceptions (:issue:`58807`) when :meth:`DataFrame.agg` is called with ``axis=1`` and a ``func`` which relabels the result index. - :class:`pandas.api.typing.FrozenList` is available for typing the outputs of :attr:`MultiIndex.names`, :attr:`MultiIndex.codes` and :attr:`MultiIndex.levels` (:issue:`58237`) - :class:`pandas.api.typing.SASReader` is available for typing the output of :func:`read_sas` (:issue:`55689`) - :func:`DataFrame.to_excel` now raises an ``UserWarning`` when the character count in a cell exceeds Excel's limitation of 32767 characters (:issue:`56954`) @@ -116,18 +117,9 @@ These improvements also fixed certain bugs in groupby: - :meth:`.DataFrameGroupBy.sum` would have incorrect values when there are multiple groupings, unobserved groups, and non-numeric data (:issue:`43891`) - :meth:`.DataFrameGroupBy.value_counts` would produce incorrect results when used with some categorical and some non-categorical groupings and ``observed=False`` (:issue:`56016`) -.. _whatsnew_300.notable_bug_fixes.improved_error_message_agg: +.. _whatsnew_300.notable_bug_fixes.notable_bug_fix2: -Improved error message for :meth:`DataFrame.agg` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When :meth:`DataFrame.agg` is called with ``axis=1`` and a ``func`` which relabels the result index, the :func:`relabel_result` function iterates over result columns rather than the result rows, causing a confusing chain of ``KeyError`` exceptions (:issue:`58807`). - -This error is now handled in :func:`frame_apply` by throwing a ``NotImplementedError`` with a more explicit error message. - -.. _whatsnew_300.notable_bug_fixes.notable_bug_fix3: - -notable_bug_fix3 +notable_bug_fix2 ^^^^^^^^^^^^^^^^ .. --------------------------------------------------------------------------- diff --git a/pandas/core/apply.py b/pandas/core/apply.py index c0b90d3148425..d024afa570a1e 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -99,8 +99,7 @@ def frame_apply( elif axis == 1: if columns: raise NotImplementedError( - "func given to frame_apply cannot contain " - "an index relabeling when axis is 1" + f"Named aggregation is not supported when {axis=}." ) klass = FrameColumnApply diff --git a/pandas/core/shared_docs.py b/pandas/core/shared_docs.py index 15c3df683f611..5725b96f66cd4 100644 --- a/pandas/core/shared_docs.py +++ b/pandas/core/shared_docs.py @@ -50,7 +50,7 @@ A passed user-defined-function will be passed a Series for evaluation. -If `func` defines an index relabeling, `axis` must be `0` or `index`. +If ``func`` defines an index relabeling, ``axis`` must be ``0`` or ``index``. {examples}""" _shared_docs["compare"] = """ diff --git a/pandas/tests/apply/test_frame_apply.py b/pandas/tests/apply/test_frame_apply.py index 4e7a2eecfcda2..fae11fd41cea8 100644 --- a/pandas/tests/apply/test_frame_apply.py +++ b/pandas/tests/apply/test_frame_apply.py @@ -1329,9 +1329,13 @@ def test_agg_reduce(axis, float_frame): expected = expected.T if axis in {1, "columns"} else expected tm.assert_frame_equal(result, expected) - msg = "func given to frame_apply cannot contain an index relabeling when axis is 1" + +def test_named_agg_reduce_axis1_raises(float_frame): + axis = 1 + name1, name2 = float_frame.axes[0].unique()[:2].sort_values() + msg = f"Named aggregation is not supported when {axis=}." with pytest.raises(NotImplementedError, match=msg): - float_frame.agg(row1=(name1, "sum"), row2=(name2, "max"), axis=1) + float_frame.agg(row1=(name1, "sum"), row2=(name2, "max"), axis=axis) def test_nuiscance_columns(): From c487cab0493b2f57d3521141a695a41da9cda5a3 Mon Sep 17 00:00:00 2001 From: Matias Lindgren Date: Mon, 15 Jul 2024 21:23:06 +0200 Subject: [PATCH 6/8] test also with named axis --- pandas/tests/apply/test_frame_apply.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/tests/apply/test_frame_apply.py b/pandas/tests/apply/test_frame_apply.py index fae11fd41cea8..78c52d3ddfbdf 100644 --- a/pandas/tests/apply/test_frame_apply.py +++ b/pandas/tests/apply/test_frame_apply.py @@ -1331,11 +1331,11 @@ def test_agg_reduce(axis, float_frame): def test_named_agg_reduce_axis1_raises(float_frame): - axis = 1 name1, name2 = float_frame.axes[0].unique()[:2].sort_values() - msg = f"Named aggregation is not supported when {axis=}." - with pytest.raises(NotImplementedError, match=msg): - float_frame.agg(row1=(name1, "sum"), row2=(name2, "max"), axis=axis) + msg = "Named aggregation is not supported when axis=1." + for axis in [1, "columns"]: + with pytest.raises(NotImplementedError, match=msg): + float_frame.agg(row1=(name1, "sum"), row2=(name2, "max"), axis=axis) def test_nuiscance_columns(): From 9fcc6a416843f9f301e3ec5b1dcfcf1a1a819293 Mon Sep 17 00:00:00 2001 From: matiaslindgren Date: Mon, 15 Jul 2024 21:24:40 +0200 Subject: [PATCH 7/8] Update doc/source/whatsnew/v3.0.0.rst Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> --- doc/source/whatsnew/v3.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 6b7f9659d7449..066184ae49baf 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -28,7 +28,7 @@ enhancement2 Other enhancements ^^^^^^^^^^^^^^^^^^ -- Fix confusing chain of ``KeyError`` exceptions (:issue:`58807`) when :meth:`DataFrame.agg` is called with ``axis=1`` and a ``func`` which relabels the result index. +- :meth:`DataFrame.agg` called with ``axis=1`` and a ``func`` which relabels the result index now raises a ``NotImplementedError` (:issue:`58807`). - :class:`pandas.api.typing.FrozenList` is available for typing the outputs of :attr:`MultiIndex.names`, :attr:`MultiIndex.codes` and :attr:`MultiIndex.levels` (:issue:`58237`) - :class:`pandas.api.typing.SASReader` is available for typing the output of :func:`read_sas` (:issue:`55689`) - :func:`DataFrame.to_excel` now raises an ``UserWarning`` when the character count in a cell exceeds Excel's limitation of 32767 characters (:issue:`56954`) From faab3b5991a47fcccf1980d8aed7d9e5f5cf0d92 Mon Sep 17 00:00:00 2001 From: Matias Lindgren Date: Mon, 15 Jul 2024 21:29:09 +0200 Subject: [PATCH 8/8] fix rst --- doc/source/whatsnew/v3.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 066184ae49baf..0a63ab9354c54 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -28,11 +28,11 @@ enhancement2 Other enhancements ^^^^^^^^^^^^^^^^^^ -- :meth:`DataFrame.agg` called with ``axis=1`` and a ``func`` which relabels the result index now raises a ``NotImplementedError` (:issue:`58807`). - :class:`pandas.api.typing.FrozenList` is available for typing the outputs of :attr:`MultiIndex.names`, :attr:`MultiIndex.codes` and :attr:`MultiIndex.levels` (:issue:`58237`) - :class:`pandas.api.typing.SASReader` is available for typing the output of :func:`read_sas` (:issue:`55689`) - :func:`DataFrame.to_excel` now raises an ``UserWarning`` when the character count in a cell exceeds Excel's limitation of 32767 characters (:issue:`56954`) - :func:`read_stata` now returns ``datetime64`` resolutions better matching those natively stored in the stata format (:issue:`55642`) +- :meth:`DataFrame.agg` called with ``axis=1`` and a ``func`` which relabels the result index now raises a ``NotImplementedError`` (:issue:`58807`). - :meth:`Styler.set_tooltips` provides alternative method to storing tooltips by using title attribute of td elements. (:issue:`56981`) - Allow dictionaries to be passed to :meth:`pandas.Series.str.replace` via ``pat`` parameter (:issue:`51748`) - Support passing a :class:`Series` input to :func:`json_normalize` that retains the :class:`Series` :class:`Index` (:issue:`51452`)