From 9832203f95c6f816d75f2d36b5030f44892bc0fe Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 1 Sep 2022 07:59:05 -0700 Subject: [PATCH 01/11] ENH: __pandas_priority__ --- pandas/core/arrays/base.py | 6 ++++++ pandas/core/frame.py | 4 ++++ pandas/core/indexes/base.py | 4 ++++ pandas/core/ops/common.py | 9 ++++----- pandas/core/series.py | 4 ++++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 1e3b137184660..aac02ea31a8ed 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -242,6 +242,12 @@ class ExtensionArray: # Don't override this. _typ = "extension" + # similar to __array_priority__, positions ExtensionArray after Index, + # Series, and DataFrame. EA subclasses may override to choose which EA + # subclass takes priority. If overriding, the value should always be + # strictly less than 2000 to be below Index.__pandas_priority__. + __pandas_priority__ = 1000 + # ------------------------------------------------------------------------ # Constructors # ------------------------------------------------------------------------ diff --git a/pandas/core/frame.py b/pandas/core/frame.py index b20decab26928..48b28f0f8cd06 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -597,6 +597,10 @@ class DataFrame(NDFrame, OpsMixin): _hidden_attrs: frozenset[str] = NDFrame._hidden_attrs | frozenset([]) _mgr: BlockManager | ArrayManager + # similar to __array_priority__, positions DataFrame before Series, Index, + # and ExtensionArray. Should NOT be overridden by subclasses. + __pandas_priority__ = 4000 + @property def _constructor(self) -> Callable[..., DataFrame]: return DataFrame diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 6da1b22b5a1dc..b2190dbf8bc37 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -333,6 +333,10 @@ class Index(IndexOpsMixin, PandasObject): # To hand over control to subclasses _join_precedence = 1 + # similar to __array_priority__, positions Index after Series and DataFrame + # but before ExtensionArray. Should NOT be overridden by subclasses. + __pandas_priority__ = 2000 + # Cython methods; see github.com/cython/cython/issues/2647 # for why we need to wrap these instead of making them class attributes # Moreover, cython will choose the appropriate-dtyped sub-function diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index f0e6aa3750cee..a60d934be2441 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -11,7 +11,6 @@ from pandas._typing import F from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndex, ABCSeries, ) @@ -61,10 +60,10 @@ def new_method(self, other): # For comparison ops, Index does *not* defer to Series pass else: - for cls in [ABCDataFrame, ABCSeries, ABCIndex]: - if isinstance(self, cls): - break - if isinstance(other, cls): + prio = getattr(other, "__pandas_priority__", None) + if prio is not None: + if prio > self.__pandas_priority__: + # e.g. other is DataFrame while self is Index/Series/EA return NotImplemented other = item_from_zerodim(other) diff --git a/pandas/core/series.py b/pandas/core/series.py index bf313925905f7..80a5d971e8673 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -326,6 +326,10 @@ class Series(base.IndexOpsMixin, NDFrame): | frozenset(["compress", "ptp"]) ) + # similar to __array_priority__, positions Series after DataFrame + # but before Index and ExtensionArray. Should NOT be overridden by subclasses. + __pandas_priority__ = 3000 + # Override cache_readonly bc Series is mutable # error: Incompatible types in assignment (expression has type "property", # base class "IndexOpsMixin" defined the type as "Callable[[IndexOpsMixin], bool]") From 0176f0e8654bfaa7e04e443265d429114c8c7217 Mon Sep 17 00:00:00 2001 From: Brock Date: Fri, 6 Jan 2023 18:47:51 -0800 Subject: [PATCH 02/11] doc, test --- doc/source/development/extending.rst | 23 +++++++++++++++++++++++ pandas/tests/test_downstream.py | 16 ++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst index c7286616672b9..f21395ed419d2 100644 --- a/doc/source/development/extending.rst +++ b/doc/source/development/extending.rst @@ -488,3 +488,26 @@ registers the default "matplotlib" backend as follows. More information on how to implement a third-party plotting backend can be found at https://github.com/pandas-dev/pandas/blob/main/pandas/plotting/__init__.py#L1. + +.. _extending.pandas_priority: + +Arithmetic with 3rd party types +------------------------------- + +In order to control how arithmetic works between a custom type and a pandas type, +implement ``__pandas_priority__``. Similar to numpy's ``__array_priority__`` +semantics, arithmetic methods on :class:`DataFrame`, :class:`Series`, and :class:`Index` +objects will return ``NotImplemented`` if the ``other`` has a higher ``__array_priority__``. + +.. code-block:: python + + class MyClass: + __pandas_priority__ = 5000 + + def __add__(self, other): + return self + + left = MyClass() + right = pd.Series([1, 2, 3]) + assert right.__add__(left) is NotImplemented + assert right + left is left diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index 2287110b63bb6..8f3002ce3aae5 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -295,3 +295,19 @@ def test_frame_setitem_dask_array_into_new_col(): tm.assert_frame_equal(result, expected) finally: pd.set_option("compute.use_numexpr", olduse) + + +def test_pandas_priority(): + # GH#48347 + + class MyClass: + __pandas_priority__ = 5000 + + def __radd__(self, other): + return self + + left = MyClass() + right = Series(range(3)) + + assert right.__add__(left) is NotImplemented + assert right + left is left From 794b7e8e0f4a910d87eeddf21d0a18e07def8747 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 22 Feb 2023 15:33:31 -0800 Subject: [PATCH 03/11] Update doc/source/development/extending.rst Co-authored-by: Marc Garcia --- doc/source/development/extending.rst | 35 ++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst index f21395ed419d2..0f41c1d7129bb 100644 --- a/doc/source/development/extending.rst +++ b/doc/source/development/extending.rst @@ -497,17 +497,38 @@ Arithmetic with 3rd party types In order to control how arithmetic works between a custom type and a pandas type, implement ``__pandas_priority__``. Similar to numpy's ``__array_priority__`` semantics, arithmetic methods on :class:`DataFrame`, :class:`Series`, and :class:`Index` -objects will return ``NotImplemented`` if the ``other`` has a higher ``__array_priority__``. +objects will delegate to ``other``, if it has an attribute ``__array_priority__`` with a higher value. + +By default, pandas objects try to operate with other objects, even if they are not types known to pandas: + +.. code-block:: python + + >>> pandas.Series([1, 2]) + [10, 20] + 0 11 + 1 22 + dtype: int64 + +In the example above, if `[10, 20]` was a custom type that can be understood as a list, pandas objects will still operate with it in the same way. + +In some cases, it is useful to delegate to the other type the operation. For example, consider I implement a +custom list object, and I want the result of adding my custom list with a pandas `Series` to be an instance of my list +and not a `Series` as seen in the previous example. This is now possible by defining the `__pandas_priority__` attribute +of my custom list, and setting it to a higher value, than the priority of the pandas objects I want to operate with. .. code-block:: python - class MyClass: + class CustomList(list): __pandas_priority__ = 5000 - def __add__(self, other): + def __radd__(self, other): + # return `self` and not the addition for simplicity return self - left = MyClass() - right = pd.Series([1, 2, 3]) - assert right.__add__(left) is NotImplemented - assert right + left is left + custom = CustomList() + series = pd.Series([1, 2, 3]) + + # Series refuses to add custom, since it's an unknown type with higher priority + assert series.__add__(custom) is NotImplemented + + # This will cause the custom class `__radd__` being used instead + assert series + custom is custom From a0ace587627dbfff7d4bf82bc1214a0a358103d9 Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 22 Feb 2023 15:35:36 -0800 Subject: [PATCH 04/11] Whatsnew --- doc/source/whatsnew/v2.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index aeaafbc4c125d..80353ecb86fd0 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -28,7 +28,7 @@ enhancement2 Other enhancements ^^^^^^^^^^^^^^^^^^ -- +- Implemented ``__pandas_priority__`` to allow custom types to take precedence over :class:`DataFrame`, :class:`Series`, :class:`Index`, or :class:`ExtensionArray` for arithmetic operations, see _extending.pandas_priority - .. --------------------------------------------------------------------------- From 8a3bdaee1c903f7aa4bdd65e344d15aee17d54db Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 22 Feb 2023 15:36:15 -0800 Subject: [PATCH 05/11] GH ref --- doc/source/whatsnew/v2.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 80353ecb86fd0..1f287785535c5 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -28,7 +28,7 @@ enhancement2 Other enhancements ^^^^^^^^^^^^^^^^^^ -- Implemented ``__pandas_priority__`` to allow custom types to take precedence over :class:`DataFrame`, :class:`Series`, :class:`Index`, or :class:`ExtensionArray` for arithmetic operations, see _extending.pandas_priority +- Implemented ``__pandas_priority__`` to allow custom types to take precedence over :class:`DataFrame`, :class:`Series`, :class:`Index`, or :class:`ExtensionArray` for arithmetic operations, see _extending.pandas_priority (:issue:`48347`) - .. --------------------------------------------------------------------------- From c86e10b3f56e59f4480bb3a630a87f78cbfc22dc Mon Sep 17 00:00:00 2001 From: Brock Date: Fri, 24 Feb 2023 13:39:03 -0800 Subject: [PATCH 06/11] suggested doc edit --- doc/source/whatsnew/v2.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 7229edb486dd3..60401a1f0fd8f 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -28,7 +28,7 @@ enhancement2 Other enhancements ^^^^^^^^^^^^^^^^^^ -- Implemented ``__pandas_priority__`` to allow custom types to take precedence over :class:`DataFrame`, :class:`Series`, :class:`Index`, or :class:`ExtensionArray` for arithmetic operations, see _extending.pandas_priority (:issue:`48347`) +- Implemented ``__pandas_priority__`` to allow custom types to take precedence over :class:`DataFrame`, :class:`Series`, :class:`Index`, or :class:`ExtensionArray` for arithmetic operations, :ref:`see the developer guide ` (:issue:`48347`) - .. --------------------------------------------------------------------------- From 05894a5d89052b5867018d5a192f475352f4f601 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 25 Feb 2023 13:16:07 -0800 Subject: [PATCH 07/11] Update doc/source/development/extending.rst Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> --- doc/source/development/extending.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst index 0f41c1d7129bb..6c5d8e5d36860 100644 --- a/doc/source/development/extending.rst +++ b/doc/source/development/extending.rst @@ -512,7 +512,7 @@ In the example above, if `[10, 20]` was a custom type that can be understood as In some cases, it is useful to delegate to the other type the operation. For example, consider I implement a custom list object, and I want the result of adding my custom list with a pandas `Series` to be an instance of my list -and not a `Series` as seen in the previous example. This is now possible by defining the `__pandas_priority__` attribute +and not a :class:`Series` as seen in the previous example. This is now possible by defining the ``__pandas_priority__`` attribute of my custom list, and setting it to a higher value, than the priority of the pandas objects I want to operate with. .. code-block:: python From 844879171c7603bf8f38106d8e9b693014a5a0d1 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 25 Feb 2023 13:16:14 -0800 Subject: [PATCH 08/11] Update doc/source/development/extending.rst Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> --- doc/source/development/extending.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst index 6c5d8e5d36860..f850ae3494f71 100644 --- a/doc/source/development/extending.rst +++ b/doc/source/development/extending.rst @@ -511,7 +511,7 @@ By default, pandas objects try to operate with other objects, even if they are n In the example above, if `[10, 20]` was a custom type that can be understood as a list, pandas objects will still operate with it in the same way. In some cases, it is useful to delegate to the other type the operation. For example, consider I implement a -custom list object, and I want the result of adding my custom list with a pandas `Series` to be an instance of my list +custom list object, and I want the result of adding my custom list with a pandas :class:`Series` to be an instance of my list and not a :class:`Series` as seen in the previous example. This is now possible by defining the ``__pandas_priority__`` attribute of my custom list, and setting it to a higher value, than the priority of the pandas objects I want to operate with. From 2d3d883f707dc6f3987aa1318c4e906c3e4124b0 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 25 Feb 2023 13:16:20 -0800 Subject: [PATCH 09/11] Update doc/source/development/extending.rst Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> --- doc/source/development/extending.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst index f850ae3494f71..d2f1b5c0bd884 100644 --- a/doc/source/development/extending.rst +++ b/doc/source/development/extending.rst @@ -508,7 +508,7 @@ By default, pandas objects try to operate with other objects, even if they are n 1 22 dtype: int64 -In the example above, if `[10, 20]` was a custom type that can be understood as a list, pandas objects will still operate with it in the same way. +In the example above, if ``[10, 20]`` was a custom type that can be understood as a list, pandas objects will still operate with it in the same way. In some cases, it is useful to delegate to the other type the operation. For example, consider I implement a custom list object, and I want the result of adding my custom list with a pandas :class:`Series` to be an instance of my list From 008e046a88aa3a1009a842078cae20962d45ae87 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 27 Feb 2023 15:37:37 -0800 Subject: [PATCH 10/11] lint fixup --- doc/source/development/extending.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst index 0f41c1d7129bb..af2a27263f2f4 100644 --- a/doc/source/development/extending.rst +++ b/doc/source/development/extending.rst @@ -503,16 +503,16 @@ By default, pandas objects try to operate with other objects, even if they are n .. code-block:: python - >>> pandas.Series([1, 2]) + [10, 20] + >>> pd.Series([1, 2]) + [10, 20] 0 11 1 22 dtype: int64 -In the example above, if `[10, 20]` was a custom type that can be understood as a list, pandas objects will still operate with it in the same way. +In the example above, if ``[10, 20]`` was a custom type that can be understood as a list, pandas objects will still operate with it in the same way. In some cases, it is useful to delegate to the other type the operation. For example, consider I implement a -custom list object, and I want the result of adding my custom list with a pandas `Series` to be an instance of my list -and not a `Series` as seen in the previous example. This is now possible by defining the `__pandas_priority__` attribute +custom list object, and I want the result of adding my custom list with a pandas ``Series`` to be an instance of my list +and not a ``Series`` as seen in the previous example. This is now possible by defining the ``__pandas_priority__`` attribute of my custom list, and setting it to a higher value, than the priority of the pandas objects I want to operate with. .. code-block:: python From be8458d2a186ba33160ccdb8c636728c0ae189ca Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 6 Mar 2023 11:30:04 -0800 Subject: [PATCH 11/11] suggested edits --- doc/source/development/extending.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/development/extending.rst b/doc/source/development/extending.rst index e6ee41541e980..1d52a5595472b 100644 --- a/doc/source/development/extending.rst +++ b/doc/source/development/extending.rst @@ -497,7 +497,7 @@ Arithmetic with 3rd party types In order to control how arithmetic works between a custom type and a pandas type, implement ``__pandas_priority__``. Similar to numpy's ``__array_priority__`` semantics, arithmetic methods on :class:`DataFrame`, :class:`Series`, and :class:`Index` -objects will delegate to ``other``, if it has an attribute ``__array_priority__`` with a higher value. +objects will delegate to ``other``, if it has an attribute ``__pandas_priority__`` with a higher value. By default, pandas objects try to operate with other objects, even if they are not types known to pandas: @@ -515,6 +515,8 @@ custom list object, and I want the result of adding my custom list with a pandas and not a :class:`Series` as seen in the previous example. This is now possible by defining the ``__pandas_priority__`` attribute of my custom list, and setting it to a higher value, than the priority of the pandas objects I want to operate with. +The ``__pandas_priority__`` of :class:`DataFrame`, :class:`Series`, and :class:`Index` are ``4000``, ``3000``, and ``2000`` respectively. The base ``ExtensionArray.__pandas_priority__`` is ``1000``. + .. code-block:: python class CustomList(list):