From 0c1eca6bc3352c16e73768b1b711c93a950917e9 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 24 May 2021 16:33:45 -0700 Subject: [PATCH 1/6] API: EA._can_hold_na -> EDtype.can_hold_na --- doc/source/whatsnew/v1.3.0.rst | 1 + pandas/core/arrays/base.py | 5 ++++- pandas/core/dtypes/base.py | 7 +++++++ pandas/core/internals/blocks.py | 10 +++++----- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 258e391b9220c..0d39b3b3ffac6 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -642,6 +642,7 @@ Other API changes - Partially initialized :class:`CategoricalDtype` (i.e. those with ``categories=None`` objects will no longer compare as equal to fully initialized dtype objects. - Accessing ``_constructor_expanddim`` on a :class:`DataFrame` and ``_constructor_sliced`` on a :class:`Series` now raise an ``AttributeError``. Previously a ``NotImplementedError`` was raised (:issue:`38782`) - Added new ``engine`` and ``**engine_kwargs`` parameters to :meth:`DataFrame.to_sql` to support other future "SQL engines". Currently we still only use ``SQLAlchemy`` under the hood, but more engines are planned to be supported such as ``turbodbc`` (:issue:`36893`) +- :attr:`ExtensionArray._can_hold_na` should now be implemented on the :class:`ExtensionDtype` subclass as ``can_hold_na`` (:issue:`40574`) Build ===== diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 7dddb9f3d6f25..97c01b1f44589 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -35,6 +35,7 @@ from pandas.util._decorators import ( Appender, Substitution, + cache_readonly, ) from pandas.util._validators import ( validate_bool_kwarg, @@ -1273,7 +1274,9 @@ def _concat_same_type( # such as take(), reindex(), shift(), etc. In addition, those results # will then be of the ExtensionArray subclass rather than an array # of objects - _can_hold_na = True + @cache_readonly + def _can_hold_na(self) -> bool: + return self.dtype.can_hold_na def _reduce(self, name: str, *, skipna: bool = True, **kwargs): """ diff --git a/pandas/core/dtypes/base.py b/pandas/core/dtypes/base.py index 9671c340a0a92..d8833e81a9714 100644 --- a/pandas/core/dtypes/base.py +++ b/pandas/core/dtypes/base.py @@ -367,6 +367,13 @@ def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None: else: return None + @property + def can_hold_na(self) -> bool: + """ + Can arrays of this dtype hold NA values? + """ + return True + def register_extension_dtype(cls: type[E]) -> type[E]: """ diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 4f1b16e747394..cc6c08c0517cc 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -173,15 +173,15 @@ def is_view(self) -> bool: return values.base is not None @final - @property + @cache_readonly def _can_hold_na(self) -> bool: """ Can we store NA values in this Block? """ - values = self.values - if isinstance(values, np.ndarray): - return values.dtype.kind not in ["b", "i", "u"] - return values._can_hold_na + dtype = self.dtype + if isinstance(dtype, np.dtype): + return dtype.kind not in ["b", "i", "u"] + return dtype.can_hold_na @final @cache_readonly From 7b151993c48dec3dc0943a049b7f3baead42a6e3 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 24 May 2021 18:44:24 -0700 Subject: [PATCH 2/6] mypy fixup --- pandas/core/arrays/categorical.py | 1 - pandas/tests/extension/test_external_block.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index cb8a08f5668ac..341a3b47c7c23 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -353,7 +353,6 @@ class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMi # tolist is not actually deprecated, just suppressed in the __dir__ _hidden_attrs = PandasObject._hidden_attrs | frozenset(["tolist"]) _typ = "categorical" - _can_hold_na = True _dtype: CategoricalDtype diff --git a/pandas/tests/extension/test_external_block.py b/pandas/tests/extension/test_external_block.py index 2402c70a166b7..18ea84305f150 100644 --- a/pandas/tests/extension/test_external_block.py +++ b/pandas/tests/extension/test_external_block.py @@ -14,9 +14,9 @@ class CustomBlock(ExtensionBlock): _holder = np.ndarray - # error: Cannot override final attribute "_can_hold_na" - # (previously declared in base class "Block") - _can_hold_na = False # type: ignore[misc] + # Incompatible types in assignment (expression has type "bool", base class + # "Block" defined the type as "Callable[[Block], bool]") + _can_hold_na = False # type: ignore[assignment,misc] @pytest.fixture From a488f2c47cdb67ea4ddca621999be379297327f0 Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 25 May 2021 07:43:15 -0700 Subject: [PATCH 3/6] mypy fixup --- pandas/core/arrays/datetimelike.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index ff46715d0a527..fbdfd3397ef4b 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -158,6 +158,9 @@ class DatetimeLikeArrayMixin(OpsMixin, NDArrayBackedExtensionArray): _is_recognized_dtype: Callable[[DtypeObj], bool] _recognized_scalars: tuple[type, ...] _ndarray: np.ndarray + # Incompatible types in assignment (expression has type "bool", base class + # "ExtensionArray" defined the type as "Callable[[ExtensionArray], bool]") + _can_hold_na = True # type: ignore[assignment] def __init__(self, data, dtype: Dtype | None = None, freq=None, copy=False): raise AbstractMethodError(self) From 5dde086fa60103f01d36d60ae3dbd6ab1e399bad Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 27 May 2021 09:26:43 -0700 Subject: [PATCH 4/6] attr-> property --- pandas/core/arrays/datetimelike.py | 7 ++++--- pandas/tests/extension/test_external_block.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index fbdfd3397ef4b..dbb592aba82f8 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -158,9 +158,10 @@ class DatetimeLikeArrayMixin(OpsMixin, NDArrayBackedExtensionArray): _is_recognized_dtype: Callable[[DtypeObj], bool] _recognized_scalars: tuple[type, ...] _ndarray: np.ndarray - # Incompatible types in assignment (expression has type "bool", base class - # "ExtensionArray" defined the type as "Callable[[ExtensionArray], bool]") - _can_hold_na = True # type: ignore[assignment] + + @cache_readonly + def _can_hold_na(self) -> bool: + return True def __init__(self, data, dtype: Dtype | None = None, freq=None, copy=False): raise AbstractMethodError(self) diff --git a/pandas/tests/extension/test_external_block.py b/pandas/tests/extension/test_external_block.py index 18ea84305f150..72222e1624dbb 100644 --- a/pandas/tests/extension/test_external_block.py +++ b/pandas/tests/extension/test_external_block.py @@ -14,9 +14,10 @@ class CustomBlock(ExtensionBlock): _holder = np.ndarray - # Incompatible types in assignment (expression has type "bool", base class - # "Block" defined the type as "Callable[[Block], bool]") - _can_hold_na = False # type: ignore[assignment,misc] + + @property + def _can_hold_na(self) -> bool: + return False @pytest.fixture From 290e7c50f28f3e3dbe8c39354bc42a719afc3766 Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 27 May 2021 14:45:30 -0700 Subject: [PATCH 5/6] mypy fixup --- pandas/tests/extension/test_external_block.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/extension/test_external_block.py b/pandas/tests/extension/test_external_block.py index 72222e1624dbb..13dec96b144ff 100644 --- a/pandas/tests/extension/test_external_block.py +++ b/pandas/tests/extension/test_external_block.py @@ -15,7 +15,8 @@ class CustomBlock(ExtensionBlock): _holder = np.ndarray - @property + # Cannot override final attribute "_can_hold_na" + @property # type: ignore[misc] def _can_hold_na(self) -> bool: return False From 253abeca23fa3e89011d83281765e85a64bbb3c4 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 7 Jun 2021 12:39:02 -0700 Subject: [PATCH 6/6] remove whatsnew --- doc/source/whatsnew/v1.3.0.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index ccbbf33023d09..8b413808503ad 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -645,7 +645,6 @@ Other API changes - Accessing ``_constructor_expanddim`` on a :class:`DataFrame` and ``_constructor_sliced`` on a :class:`Series` now raise an ``AttributeError``. Previously a ``NotImplementedError`` was raised (:issue:`38782`) - Added new ``engine`` and ``**engine_kwargs`` parameters to :meth:`DataFrame.to_sql` to support other future "SQL engines". Currently we still only use ``SQLAlchemy`` under the hood, but more engines are planned to be supported such as ``turbodbc`` (:issue:`36893`) - Removed redundant ``freq`` from :class:`PeriodIndex` string representation (:issue:`41653`) -- :attr:`ExtensionArray._can_hold_na` should now be implemented on the :class:`ExtensionDtype` subclass as ``_can_hold_na`` (:issue:`40574`) Build =====