From fef8aa16b899e17caca97f63a97ddb2384e4d6d3 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 21 Jun 2023 11:00:49 +0100 Subject: [PATCH 1/8] DEPR: deprecate returning a tuple from a callable in iloc indexing The current semantics are that tuple-destructuring of the key is performed before unwinding any callables. As such, if a callable returns a tuple for iloc, it will be handled incorrectly. To avoid this, explicitly deprecate support for this behaviour. Closes #53533. --- pandas/core/indexing.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 4a2803f638c73..381f9c6ab0897 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -27,6 +27,7 @@ _chained_assignment_msg, ) from pandas.util._decorators import doc +from pandas.util._exceptions import find_stack_level from pandas.core.dtypes.cast import ( can_hold_element, @@ -869,7 +870,19 @@ def __setitem__(self, key, value) -> None: key = tuple(list(x) if is_iterator(x) else x for x in key) key = tuple(com.apply_if_callable(x, self.obj) for x in key) else: - key = com.apply_if_callable(key, self.obj) + maybe_callable = com.apply_if_callable(key, self.obj) + if ( + self.name == "iloc" + and callable(key) + and isinstance(maybe_callable, tuple) + ): + warnings.warn( + "Returning a tuple from a callable in iLocation indexing " + "is deprecated and will be removed in a future version", + FutureWarning, + stacklevel=find_stack_level(), + ) + key = maybe_callable indexer = self._get_setitem_indexer(key) self._has_valid_setitem_indexer(key) @@ -1130,6 +1143,17 @@ def __getitem__(self, key): axis = self.axis or 0 maybe_callable = com.apply_if_callable(key, self.obj) + if ( + self.name == "iloc" + and callable(key) + and isinstance(maybe_callable, tuple) + ): + warnings.warn( + "Returning a tuple from a callable in iLocation indexing " + "is deprecated and will be removed in a future version", + FutureWarning, + stacklevel=find_stack_level(), + ) return self._getitem_axis(maybe_callable, axis=axis) def _is_scalar_access(self, key: tuple): From 268f5e53782b8afb7d1acfc5e5e6afcc312a503b Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Wed, 21 Jun 2023 12:11:42 +0100 Subject: [PATCH 2/8] Update documentation and add test --- doc/source/user_guide/indexing.rst | 18 ++++++++++++++++++ doc/source/whatsnew/v2.1.0.rst | 1 + pandas/core/indexing.py | 3 ++- pandas/tests/frame/indexing/test_indexing.py | 8 ++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/doc/source/user_guide/indexing.rst b/doc/source/user_guide/indexing.rst index 77eee8e58a5e8..e514d3c320265 100644 --- a/doc/source/user_guide/indexing.rst +++ b/doc/source/user_guide/indexing.rst @@ -62,6 +62,8 @@ of multi-axis indexing. * A boolean array (any ``NA`` values will be treated as ``False``). * A ``callable`` function with one argument (the calling Series or DataFrame) and that returns valid output for indexing (one of the above). + * A tuple of row (and column) indices whose elements are one of the + above inputs. See more at :ref:`Selection by Label `. @@ -78,6 +80,8 @@ of multi-axis indexing. * A boolean array (any ``NA`` values will be treated as ``False``). * A ``callable`` function with one argument (the calling Series or DataFrame) and that returns valid output for indexing (one of the above). + * A tuple of row (and column) indices whose elements are one of the + above inputs. See more at :ref:`Selection by Position `, :ref:`Advanced Indexing ` and :ref:`Advanced @@ -85,6 +89,12 @@ of multi-axis indexing. * ``.loc``, ``.iloc``, and also ``[]`` indexing can accept a ``callable`` as indexer. See more at :ref:`Selection By Callable `. + .. note:: + + Destructuring tuple keys into row (and column) indexes occurs + *before* callables are applied, so you cannot return a tuple from + a callable to index both rows and columns. + Getting values from an object with multi-axes selection uses the following notation (using ``.loc`` as an example, but the following applies to ``.iloc`` as well). Any of the axes accessors may be the null slice ``:``. Axes left out of @@ -446,6 +456,8 @@ The ``.iloc`` attribute is the primary access method. The following are valid in * A slice object with ints ``1:7``. * A boolean array. * A ``callable``, see :ref:`Selection By Callable `. +* A tuple of row (and column) indexes, whose elements are one of the + above types. .. ipython:: python @@ -547,6 +559,12 @@ Selection by callable ``.loc``, ``.iloc``, and also ``[]`` indexing can accept a ``callable`` as indexer. The ``callable`` must be a function with one argument (the calling Series or DataFrame) that returns valid output for indexing. +.. note:: + + For ``.iloc`` indexing, returning a tuple from the callable is + not supported, since tuple destructuring for row and column indexes + occurs *before* applying callables. + .. ipython:: python df1 = pd.DataFrame(np.random.randn(6, 4), diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 44691e4265f5b..69f214ea96d03 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -298,6 +298,7 @@ Deprecations - Deprecated parameter ``obj`` in :meth:`GroupBy.get_group` (:issue:`53545`) - Deprecated positional indexing on :class:`Series` with :meth:`Series.__getitem__` and :meth:`Series.__setitem__`, in a future version ``ser[item]`` will *always* interpret ``item`` as a label, not a position (:issue:`50617`) - Deprecated strings ``T``, ``t``, ``L`` and ``l`` denoting units in :func:`to_timedelta` (:issue:`52536`) +- Deprecated support for returning a tuple from a callable in ``iloc``-indexing. Place callables inside a tuple if you need to generate row and column indices using functions (:issue:`53533`) - Deprecated the "method" and "limit" keywords on :meth:`Series.fillna`, :meth:`DataFrame.fillna`, :meth:`SeriesGroupBy.fillna`, :meth:`DataFrameGroupBy.fillna`, and :meth:`Resampler.fillna`, use ``obj.bfill()`` or ``obj.ffill()`` instead (:issue:`53394`) - Deprecated the ``method`` and ``limit`` keywords in :meth:`DataFrame.replace` and :meth:`Series.replace` (:issue:`33302`) - Deprecated values "pad", "ffill", "bfill", "backfill" for :meth:`Series.interpolate` and :meth:`DataFrame.interpolate`, use ``obj.ffill()`` or ``obj.bfill()`` instead (:issue:`53581`) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 381f9c6ab0897..5666c932c712b 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -163,7 +163,8 @@ def iloc(self) -> _iLocIndexer: - A ``callable`` function with one argument (the calling Series or DataFrame) and that returns valid output for indexing (one of the above). This is useful in method chains, when you don't have a reference to the - calling object, but would like to base your selection on some value. + calling object, but would like to base your selection on + some value. Note that returning a tuple from a callable is deprecated. - A tuple of row and column indexes. The tuple elements consist of one of the above inputs, e.g. ``(0, 1)``. diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index 9005798d66d17..159c7ec18a2b0 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -1000,6 +1000,14 @@ def test_single_element_ix_dont_upcast(self, float_frame): result = df.loc[[0], "b"] tm.assert_series_equal(result, expected) + def test_iloc_callable_tuple_return_value(self): + df = DataFrame(np.random.randn(10, 4), index=range(0, 20, 2)) + msg = "callable in iLocation indexing is deprecated" + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[lambda _: (0,)] + with tm.assert_produces_warning(FutureWarning, match=msg): + df.iloc[lambda _: (0,)] = 1 + def test_iloc_row(self): df = DataFrame(np.random.randn(10, 4), index=range(0, 20, 2)) From c55e2c329d0ce432eccdeadd9364ac01c5085e98 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Thu, 22 Jun 2023 10:35:29 +0100 Subject: [PATCH 3/8] Refactor check into method --- pandas/core/indexing.py | 46 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 5666c932c712b..ee24965ef45b6 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -4,8 +4,10 @@ import sys from typing import ( TYPE_CHECKING, + Any, Hashable, Sequence, + TypeVar, cast, final, ) @@ -87,6 +89,7 @@ Series, ) +T = TypeVar("T") # "null slice" _NS = slice(None, None) _one_ellipsis_message = "indexer may only contain one '...' entry" @@ -164,7 +167,12 @@ def iloc(self) -> _iLocIndexer: DataFrame) and that returns valid output for indexing (one of the above). This is useful in method chains, when you don't have a reference to the calling object, but would like to base your selection on - some value. Note that returning a tuple from a callable is deprecated. + some value. + + .. deprecated:: 2.1.0 + + Returning a tuple from a callable is deprecated. + - A tuple of row and column indexes. The tuple elements consist of one of the above inputs, e.g. ``(0, 1)``. @@ -872,18 +880,7 @@ def __setitem__(self, key, value) -> None: key = tuple(com.apply_if_callable(x, self.obj) for x in key) else: maybe_callable = com.apply_if_callable(key, self.obj) - if ( - self.name == "iloc" - and callable(key) - and isinstance(maybe_callable, tuple) - ): - warnings.warn( - "Returning a tuple from a callable in iLocation indexing " - "is deprecated and will be removed in a future version", - FutureWarning, - stacklevel=find_stack_level(), - ) - key = maybe_callable + key = self._check_deprecated_callable_usage(key, maybe_callable) indexer = self._get_setitem_indexer(key) self._has_valid_setitem_indexer(key) @@ -1130,6 +1127,17 @@ def _getitem_nested_tuple(self, tup: tuple): def _convert_to_indexer(self, key, axis: AxisInt): raise AbstractMethodError(self) + def _check_deprecated_callable_usage(self, key: Any, maybe_callable: T) -> T: + # GH53533 + if self.name == "iloc" and callable(key) and isinstance(maybe_callable, tuple): + warnings.warn( + "Returning a tuple from a callable in iLocation indexing " + "is deprecated and will be removed in a future version", + FutureWarning, + stacklevel=find_stack_level(), + ) + return maybe_callable + @final def __getitem__(self, key): check_dict_or_set_indexers(key) @@ -1144,17 +1152,7 @@ def __getitem__(self, key): axis = self.axis or 0 maybe_callable = com.apply_if_callable(key, self.obj) - if ( - self.name == "iloc" - and callable(key) - and isinstance(maybe_callable, tuple) - ): - warnings.warn( - "Returning a tuple from a callable in iLocation indexing " - "is deprecated and will be removed in a future version", - FutureWarning, - stacklevel=find_stack_level(), - ) + maybe_callable = self._check_deprecated_callable_usage(key, maybe_callable) return self._getitem_axis(maybe_callable, axis=axis) def _is_scalar_access(self, key: tuple): From d2004ede31139d1ac217aeba56cef02afd409d47 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Thu, 22 Jun 2023 10:35:46 +0100 Subject: [PATCH 4/8] Link to docs in whatsnew entry --- doc/source/whatsnew/v2.1.0.rst | 2 +- pandas/tests/frame/indexing/test_indexing.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 69f214ea96d03..df351c7da3d6d 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -298,7 +298,7 @@ Deprecations - Deprecated parameter ``obj`` in :meth:`GroupBy.get_group` (:issue:`53545`) - Deprecated positional indexing on :class:`Series` with :meth:`Series.__getitem__` and :meth:`Series.__setitem__`, in a future version ``ser[item]`` will *always* interpret ``item`` as a label, not a position (:issue:`50617`) - Deprecated strings ``T``, ``t``, ``L`` and ``l`` denoting units in :func:`to_timedelta` (:issue:`52536`) -- Deprecated support for returning a tuple from a callable in ``iloc``-indexing. Place callables inside a tuple if you need to generate row and column indices using functions (:issue:`53533`) +- Deprecated support for returning a tuple from a callable in ``iloc``-indexing. See :ref:`Selection by Position ` for details (:issue:`53533`) - Deprecated the "method" and "limit" keywords on :meth:`Series.fillna`, :meth:`DataFrame.fillna`, :meth:`SeriesGroupBy.fillna`, :meth:`DataFrameGroupBy.fillna`, and :meth:`Resampler.fillna`, use ``obj.bfill()`` or ``obj.ffill()`` instead (:issue:`53394`) - Deprecated the ``method`` and ``limit`` keywords in :meth:`DataFrame.replace` and :meth:`Series.replace` (:issue:`33302`) - Deprecated values "pad", "ffill", "bfill", "backfill" for :meth:`Series.interpolate` and :meth:`DataFrame.interpolate`, use ``obj.ffill()`` or ``obj.bfill()`` instead (:issue:`53581`) diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index 159c7ec18a2b0..e1d67cc9de7eb 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -1001,6 +1001,7 @@ def test_single_element_ix_dont_upcast(self, float_frame): tm.assert_series_equal(result, expected) def test_iloc_callable_tuple_return_value(self): + # GH53769 df = DataFrame(np.random.randn(10, 4), index=range(0, 20, 2)) msg = "callable in iLocation indexing is deprecated" with tm.assert_produces_warning(FutureWarning, match=msg): From 84573c2fb169082dcf6b269ed4b897cdb1e9d4b4 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Mon, 26 Jun 2023 09:58:00 +0100 Subject: [PATCH 5/8] Move deprecation warning so docstring checks don't complain --- pandas/core/indexing.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index ee24965ef45b6..e9bab1e87ead4 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -153,6 +153,10 @@ def iloc(self) -> _iLocIndexer: """ Purely integer-location based indexing for selection by position. + .. deprecated:: 2.1.0 + + Returning a tuple from a callable is deprecated. + ``.iloc[]`` is primarily integer position based (from ``0`` to ``length-1`` of the axis), but may also be used with a boolean array. @@ -168,11 +172,6 @@ def iloc(self) -> _iLocIndexer: This is useful in method chains, when you don't have a reference to the calling object, but would like to base your selection on some value. - - .. deprecated:: 2.1.0 - - Returning a tuple from a callable is deprecated. - - A tuple of row and column indexes. The tuple elements consist of one of the above inputs, e.g. ``(0, 1)``. @@ -193,7 +192,7 @@ def iloc(self) -> _iLocIndexer: -------- >>> mydict = [{'a': 1, 'b': 2, 'c': 3, 'd': 4}, ... {'a': 100, 'b': 200, 'c': 300, 'd': 400}, - ... {'a': 1000, 'b': 2000, 'c': 3000, 'd': 4000 }] + ... {'a': 1000, 'b': 2000, 'c': 3000, 'd': 4000}] >>> df = pd.DataFrame(mydict) >>> df a b c d From 822aca8d62800b8fa4542e50e13a4e151e719cd0 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:32:44 -0800 Subject: [PATCH 6/8] Prelim changes --- doc/source/whatsnew/v2.1.0.rst | 1 - pandas/core/indexing.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index df351c7da3d6d..44691e4265f5b 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -298,7 +298,6 @@ Deprecations - Deprecated parameter ``obj`` in :meth:`GroupBy.get_group` (:issue:`53545`) - Deprecated positional indexing on :class:`Series` with :meth:`Series.__getitem__` and :meth:`Series.__setitem__`, in a future version ``ser[item]`` will *always* interpret ``item`` as a label, not a position (:issue:`50617`) - Deprecated strings ``T``, ``t``, ``L`` and ``l`` denoting units in :func:`to_timedelta` (:issue:`52536`) -- Deprecated support for returning a tuple from a callable in ``iloc``-indexing. See :ref:`Selection by Position ` for details (:issue:`53533`) - Deprecated the "method" and "limit" keywords on :meth:`Series.fillna`, :meth:`DataFrame.fillna`, :meth:`SeriesGroupBy.fillna`, :meth:`DataFrameGroupBy.fillna`, and :meth:`Resampler.fillna`, use ``obj.bfill()`` or ``obj.ffill()`` instead (:issue:`53394`) - Deprecated the ``method`` and ``limit`` keywords in :meth:`DataFrame.replace` and :meth:`Series.replace` (:issue:`33302`) - Deprecated values "pad", "ffill", "bfill", "backfill" for :meth:`Series.interpolate` and :meth:`DataFrame.interpolate`, use ``obj.ffill()`` or ``obj.bfill()`` instead (:issue:`53581`) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index e9bab1e87ead4..471d5ad185dd6 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -153,7 +153,7 @@ def iloc(self) -> _iLocIndexer: """ Purely integer-location based indexing for selection by position. - .. deprecated:: 2.1.0 + .. deprecated:: 2.2.0 Returning a tuple from a callable is deprecated. From 67ff0d61862fb241fd1aed192ba2b7fea2798f1b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:36:14 -0800 Subject: [PATCH 7/8] Update pandas/core/indexing.py --- pandas/core/indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 8443c38f73ce8..8be4dd1247899 100644 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1151,7 +1151,7 @@ def _check_deprecated_callable_usage(self, key: Any, maybe_callable: T) -> T: # GH53533 if self.name == "iloc" and callable(key) and isinstance(maybe_callable, tuple): warnings.warn( - "Returning a tuple from a callable in iLocation indexing " + "Returning a tuple from a callable with iloc " "is deprecated and will be removed in a future version", FutureWarning, stacklevel=find_stack_level(), From 269fe0df35b612873988fcd6985a3744c86d328a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 7 Nov 2023 07:59:22 -0800 Subject: [PATCH 8/8] Adjust test --- pandas/tests/frame/indexing/test_indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py index 050ab90ae048a..765671fdd7f25 100644 --- a/pandas/tests/frame/indexing/test_indexing.py +++ b/pandas/tests/frame/indexing/test_indexing.py @@ -1020,7 +1020,7 @@ def test_single_element_ix_dont_upcast(self, float_frame): def test_iloc_callable_tuple_return_value(self): # GH53769 df = DataFrame(np.arange(40).reshape(10, 4), index=range(0, 20, 2)) - msg = "callable in iLocation indexing is deprecated" + msg = "callable with iloc" with tm.assert_produces_warning(FutureWarning, match=msg): df.iloc[lambda _: (0,)] with tm.assert_produces_warning(FutureWarning, match=msg):