From dda21d17c35e95c47a1439188d83970c1e8f4372 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Sat, 1 May 2021 15:36:08 -0400 Subject: [PATCH 1/7] TYP: Typing changes for ExtensionArray.astype --- pandas/_testing/asserters.py | 10 +++++++--- pandas/core/arrays/base.py | 16 +++++++++++++--- pandas/core/arrays/boolean.py | 17 +++++++++++++++-- pandas/core/arrays/categorical.py | 9 +++++++++ pandas/core/arrays/floating.py | 13 ++++++++++++- pandas/core/arrays/integer.py | 12 +++++++++++- pandas/core/arrays/masked.py | 9 +++++++++ pandas/core/groupby/groupby.py | 2 +- pandas/core/internals/blocks.py | 4 +++- 9 files changed, 80 insertions(+), 12 deletions(-) diff --git a/pandas/_testing/asserters.py b/pandas/_testing/asserters.py index 912039b7571bc..723b09bccfe64 100644 --- a/pandas/_testing/asserters.py +++ b/pandas/_testing/asserters.py @@ -395,9 +395,13 @@ def _get_ilevel_values(index, level): # skip exact index checking when `check_categorical` is False if check_exact and check_categorical: if not left.equals(right): - diff = ( - np.sum((left._values != right._values).astype(int)) * 100.0 / len(left) - ) + # error: Value of type variable "_Number" of "sum" cannot be + # "Union[ExtensionArray, ndarray, Any]" + thesum = np.sum( + (left._values != right._values).astype(int) + ) # type: ignore[type-var] + # error: Unsupported operand types for * ("ExtensionArray" and "float") + diff = thesum * 100.0 / len(left) # type: ignore[operator] msg = f"{obj} values are different ({np.round(diff, 5)} %)" raise_assert_detail(obj, msg, left, right) else: diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 5a2643dd531ed..78d1345d862d3 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -16,6 +16,7 @@ Sequence, TypeVar, cast, + overload, ) import numpy as np @@ -24,6 +25,7 @@ from pandas._typing import ( ArrayLike, Dtype, + NpDtype, PositionalIndexer, Shape, ) @@ -511,7 +513,15 @@ def nbytes(self) -> int: # Additional Methods # ------------------------------------------------------------------------ - def astype(self, dtype, copy=True): + @overload + def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + ... + + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: """ Cast to a NumPy array with 'dtype'. @@ -545,7 +555,7 @@ def astype(self, dtype, copy=True): ): # allow conversion to StringArrays return dtype.construct_array_type()._from_sequence(self, copy=False) - return np.array(self, dtype=dtype, copy=copy) + return np.array(self, dtype=cast(NpDtype, dtype), copy=copy) def isna(self) -> np.ndarray | ExtensionArraySupportsAnyAll: """ @@ -925,7 +935,7 @@ def _values_for_factorize(self) -> tuple[np.ndarray, Any]: The values returned by this method are also used in :func:`pandas.util.hash_pandas_object`. """ - return self.astype(object), np.nan + return cast(np.ndarray, self.astype(object)), np.nan def factorize(self, na_sentinel: int = -1) -> tuple[np.ndarray, ExtensionArray]: """ diff --git a/pandas/core/arrays/boolean.py b/pandas/core/arrays/boolean.py index 14d059c04b7c0..c05040da1d49e 100644 --- a/pandas/core/arrays/boolean.py +++ b/pandas/core/arrays/boolean.py @@ -1,7 +1,10 @@ from __future__ import annotations import numbers -from typing import TYPE_CHECKING +from typing import ( + TYPE_CHECKING, + overload, +) import warnings import numpy as np @@ -13,6 +16,7 @@ from pandas._typing import ( ArrayLike, Dtype, + NpDtype, type_t, ) from pandas.compat.numpy import function as nv @@ -392,7 +396,16 @@ def reconstruct(x): def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value) - def astype(self, dtype, copy: bool = True) -> ArrayLike: + @overload + def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + ... + + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + """ Cast to a NumPy array or ExtensionArray with 'dtype'. diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 6f3643e80a0fa..c8efec1ae7f94 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -11,6 +11,7 @@ TypeVar, Union, cast, + overload, ) from warnings import ( catch_warnings, @@ -481,6 +482,14 @@ def _constructor(self) -> type[Categorical]: def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy=False): return Categorical(scalars, dtype=dtype, copy=copy) + @overload + def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + ... + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: """ Coerce this type to another dtype diff --git a/pandas/core/arrays/floating.py b/pandas/core/arrays/floating.py index 1acbcf17dfffd..35dc8cec96cc3 100644 --- a/pandas/core/arrays/floating.py +++ b/pandas/core/arrays/floating.py @@ -1,5 +1,6 @@ from __future__ import annotations +from typing import overload import warnings import numpy as np @@ -10,7 +11,9 @@ ) from pandas._typing import ( ArrayLike, + Dtype, DtypeObj, + NpDtype, ) from pandas.compat.numpy import function as nv from pandas.util._decorators import cache_readonly @@ -271,7 +274,15 @@ def _from_sequence_of_strings( def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value, dtype=self.dtype) - def astype(self, dtype, copy: bool = True) -> ArrayLike: + @overload + def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + ... + + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: """ Cast to a NumPy array or ExtensionArray with 'dtype'. diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index b533018cdfa6b..fe9c8aa62683b 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -1,5 +1,6 @@ from __future__ import annotations +from typing import overload import warnings import numpy as np @@ -13,6 +14,7 @@ ArrayLike, Dtype, DtypeObj, + NpDtype, ) from pandas.compat.numpy import function as nv from pandas.util._decorators import cache_readonly @@ -335,7 +337,15 @@ def _from_sequence_of_strings( def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value, dtype=self.dtype) - def astype(self, dtype, copy: bool = True) -> ArrayLike: + @overload + def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + ... + + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: """ Cast to a NumPy array or ExtensionArray with 'dtype'. diff --git a/pandas/core/arrays/masked.py b/pandas/core/arrays/masked.py index 11f9f645920ec..0e813cde0462b 100644 --- a/pandas/core/arrays/masked.py +++ b/pandas/core/arrays/masked.py @@ -5,6 +5,7 @@ Any, Sequence, TypeVar, + overload, ) import numpy as np @@ -301,6 +302,14 @@ def to_numpy( # type: ignore[override] data = self._data.astype(dtype, copy=copy) return data + @overload + def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + ... + def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: dtype = pandas_dtype(dtype) diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index 7fe9d7cb49eb5..cdc26b3b3bdb8 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -1401,7 +1401,7 @@ def _bool_agg(self, val_test, skipna): Shared func to call any / all Cython GroupBy implementations. """ - def objs_to_bool(vals: ArrayLike) -> tuple[np.ndarray, type]: + def objs_to_bool(vals: ArrayLike) -> tuple[ArrayLike, type]: if is_object_dtype(vals): vals = np.array([bool(x) for x in vals]) elif isinstance(vals, BaseMaskedArray): diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 61396fdf372d5..ba81f310f69e7 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1354,7 +1354,9 @@ def get_values(self, dtype: DtypeObj | None = None) -> np.ndarray: """ values = self.values if dtype == _dtype_obj: - values = values.astype(object) + # error: Incompatible types in assignment (expression has type + # "Union[ExtensionArray, ndarray]", variable has type "ExtensionArray") + values = values.astype(object) # type: ignore[assignment] # TODO(EA2D): reshape not needed with 2D EAs return np.asarray(values).reshape(self.shape) From ed106fc5f56a6e628a663802170c0dc7eb19efdf Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Sun, 23 May 2021 12:38:48 -0400 Subject: [PATCH 2/7] Change NpDtype --- pandas/_testing/asserters.py | 9 ++------- pandas/_typing.py | 6 ++---- pandas/core/arrays/base.py | 10 ++++------ pandas/core/arrays/categorical.py | 6 +----- pandas/core/arrays/masked.py | 4 +--- pandas/core/arrays/sparse/array.py | 4 +--- pandas/core/common.py | 7 +------ pandas/core/construction.py | 4 +--- pandas/core/dtypes/dtypes.py | 3 +-- pandas/core/groupby/groupby.py | 2 +- pandas/core/internals/blocks.py | 6 ++---- 11 files changed, 17 insertions(+), 44 deletions(-) diff --git a/pandas/_testing/asserters.py b/pandas/_testing/asserters.py index 04363db64fbf8..bfc5c0d9fa811 100644 --- a/pandas/_testing/asserters.py +++ b/pandas/_testing/asserters.py @@ -399,13 +399,8 @@ def _get_ilevel_values(index, level): # skip exact index checking when `check_categorical` is False if check_exact and check_categorical: if not left.equals(right): - # error: Value of type variable "_Number" of "sum" cannot be - # "Union[ExtensionArray, ndarray, Any]" - thesum = np.sum( - (left._values != right._values).astype(int) - ) # type: ignore[type-var] - # error: Unsupported operand types for * ("ExtensionArray" and "float") - diff = thesum * 100.0 / len(left) # type: ignore[operator] + thesum = np.sum((left._values != right._values).astype(int)) + diff = thesum * 100.0 / len(left) msg = f"{obj} values are different ({np.round(diff, 5)} %)" raise_assert_detail(obj, msg, left, right) else: diff --git a/pandas/_typing.py b/pandas/_typing.py index 7763b0ceb610a..5a7df198702b0 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -124,10 +124,8 @@ Axes = Collection[Any] # dtypes -NpDtype = Union[str, np.dtype] -Dtype = Union[ - "ExtensionDtype", NpDtype, type_t[Union[str, float, int, complex, bool, object]] -] +NpDtype = Union[str, np.dtype, type_t[Union[str, float, int, complex, bool, object]]] +Dtype = Union["ExtensionDtype", NpDtype] # DtypeArg specifies all allowable dtypes in a functions its dtype argument DtypeArg = Union[Dtype, Dict[Hashable, Dtype]] DtypeObj = Union[np.dtype, "ExtensionDtype"] diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 1e2b6fae7425a..fd6f35a9b91da 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -518,16 +518,16 @@ def nbytes(self) -> int: # ------------------------------------------------------------------------ @overload - def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: ... @overload - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: ... def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: """ - Cast to a NumPy array with 'dtype'. + Cast to a NumPy array or ExtensionArray with 'dtype'. Parameters ---------- @@ -947,9 +947,7 @@ def _values_for_factorize(self) -> tuple[np.ndarray, Any]: The values returned by this method are also used in :func:`pandas.util.hash_pandas_object`. """ - # error: Incompatible return value type (got "Tuple[Union[ExtensionArray, - # ndarray], float]", expected "Tuple[ndarray, Any]") - return self.astype(object), np.nan # type: ignore[return-value] + return self.astype(object), np.nan def factorize(self, na_sentinel: int = -1) -> tuple[np.ndarray, ExtensionArray]: """ diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index bc52ad5151eb1..7c415a6398b54 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -2477,11 +2477,7 @@ def _str_get_dummies(self, sep="|"): # sep may not be in categories. Just bail on this. from pandas.core.arrays import PandasArray - # error: Argument 1 to "PandasArray" has incompatible type - # "ExtensionArray"; expected "Union[ndarray, PandasArray]" - return PandasArray(self.astype(str))._str_get_dummies( # type: ignore[arg-type] - sep - ) + return PandasArray(self.astype(str))._str_get_dummies(sep) # The Series.cat accessor diff --git a/pandas/core/arrays/masked.py b/pandas/core/arrays/masked.py index 0e813cde0462b..c8aee11b36b44 100644 --- a/pandas/core/arrays/masked.py +++ b/pandas/core/arrays/masked.py @@ -281,9 +281,7 @@ def to_numpy( # type: ignore[override] if na_value is lib.no_default: na_value = libmissing.NA if dtype is None: - # error: Incompatible types in assignment (expression has type - # "Type[object]", variable has type "Union[str, dtype[Any], None]") - dtype = object # type: ignore[assignment] + dtype = object if self._hasna: if ( not is_object_dtype(dtype) diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py index 4847372f18239..7b236095cd825 100644 --- a/pandas/core/arrays/sparse/array.py +++ b/pandas/core/arrays/sparse/array.py @@ -523,9 +523,7 @@ def __array__(self, dtype: NpDtype | None = None) -> np.ndarray: try: dtype = np.result_type(self.sp_values.dtype, type(fill_value)) except TypeError: - # error: Incompatible types in assignment (expression has type - # "Type[object]", variable has type "Union[str, dtype[Any], None]") - dtype = object # type: ignore[assignment] + dtype = object out = np.full(self.shape, fill_value, dtype=dtype) out[self.sp_index.to_int_index().indices] = self.sp_values diff --git a/pandas/core/common.py b/pandas/core/common.py index 04ff2d2c4618f..1296a6b1447c7 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -229,12 +229,7 @@ def asarray_tuplesafe(values, dtype: NpDtype | None = None) -> np.ndarray: # expected "ndarray") return values._values # type: ignore[return-value] - # error: Non-overlapping container check (element type: "Union[str, dtype[Any], - # None]", container item type: "type") - if isinstance(values, list) and dtype in [ # type: ignore[comparison-overlap] - np.object_, - object, - ]: + if isinstance(values, list) and dtype in [np.object_, object]: return construct_1d_object_array_from_listlike(values) result = np.asarray(values, dtype=dtype) diff --git a/pandas/core/construction.py b/pandas/core/construction.py index 51b9ed5fd22c7..fa6579e15a635 100644 --- a/pandas/core/construction.py +++ b/pandas/core/construction.py @@ -588,9 +588,7 @@ def _sanitize_ndim( if is_object_dtype(dtype) and isinstance(dtype, ExtensionDtype): # i.e. PandasDtype("O") - # error: Argument "dtype" to "asarray_tuplesafe" has incompatible type - # "Type[object]"; expected "Union[str, dtype[Any], None]" - result = com.asarray_tuplesafe(data, dtype=object) # type: ignore[arg-type] + result = com.asarray_tuplesafe(data, dtype=object) cls = dtype.construct_array_type() result = cls._from_sequence(result, dtype=dtype) else: diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py index c5efd8f77495c..458905ed862eb 100644 --- a/pandas/core/dtypes/dtypes.py +++ b/pandas/core/dtypes/dtypes.py @@ -29,7 +29,6 @@ from pandas._typing import ( Dtype, DtypeObj, - NpDtype, Ordered, type_t, ) @@ -1291,7 +1290,7 @@ class PandasDtype(ExtensionDtype): _metadata = ("_dtype",) - def __init__(self, dtype: NpDtype | PandasDtype | None): + def __init__(self, dtype: str | np.dtype | PandasDtype | None): if isinstance(dtype, PandasDtype): # make constructor univalent dtype = dtype.numpy_dtype diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index 0220874b44f8d..b27eb4bb8f325 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -1477,7 +1477,7 @@ def _bool_agg(self, val_test, skipna): Shared func to call any / all Cython GroupBy implementations. """ - def objs_to_bool(vals: ArrayLike) -> tuple[ArrayLike, type]: + def objs_to_bool(vals: ArrayLike) -> tuple[np.ndarray, type]: if is_object_dtype(vals): vals = np.array([bool(x) for x in vals]) elif isinstance(vals, BaseMaskedArray): diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index c8ba9f327624b..1e6f1b0c41fd2 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -1348,11 +1348,9 @@ def get_values(self, dtype: DtypeObj | None = None) -> np.ndarray: """ return object dtype as boxed values, such as Timestamps/Timedelta """ - values = self.values + values: ArrayLike = self.values if dtype == _dtype_obj: - # error: Incompatible types in assignment (expression has type - # "Union[ExtensionArray, ndarray]", variable has type "ExtensionArray") - values = values.astype(object) # type: ignore[assignment] + values = values.astype(object) # TODO(EA2D): reshape not needed with 2D EAs return np.asarray(values).reshape(self.shape) From d9ef38f3e5dc5b486635d63040b86c4cada4df42 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Mon, 31 May 2021 11:07:41 -0400 Subject: [PATCH 3/7] doc changes. Use np.dtype(object) --- pandas/core/arrays/base.py | 5 +++-- pandas/core/construction.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index fd6f35a9b91da..2c2e56273a695 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -540,8 +540,9 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: Returns ------- - array : ArrayLike - An ExtensionArray if dtype StringDtype or same as that of underlying array. + array : np.ndarray or ExtensionArray + An ExtensionArray if dtype is StringDtype, + or same as that of underlying array. Otherwise a NumPy ndarray with 'dtype' for its dtype. """ from pandas.core.arrays.string_ import StringDtype diff --git a/pandas/core/construction.py b/pandas/core/construction.py index dca75ad0d48a5..ff92dcc978616 100644 --- a/pandas/core/construction.py +++ b/pandas/core/construction.py @@ -620,7 +620,7 @@ def _sanitize_ndim( if is_object_dtype(dtype) and isinstance(dtype, ExtensionDtype): # i.e. PandasDtype("O") - result = com.asarray_tuplesafe(data, dtype=object) + result = com.asarray_tuplesafe(data, dtype=np.dtype(object)) cls = dtype.construct_array_type() result = cls._from_sequence(result, dtype=dtype) else: From e683efe0272a24938b1592d0fd50d676b66eacec Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Tue, 1 Jun 2021 09:00:22 -0400 Subject: [PATCH 4/7] fix indentation in docstring --- pandas/core/arrays/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 2c2e56273a695..8df95ad53142a 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -542,7 +542,7 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: ------- array : np.ndarray or ExtensionArray An ExtensionArray if dtype is StringDtype, - or same as that of underlying array. + or same as that of underlying array. Otherwise a NumPy ndarray with 'dtype' for its dtype. """ from pandas.core.arrays.string_ import StringDtype From a61e7ae184f20c288d49a1a49208a92f9f1585f1 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Mon, 5 Jul 2021 13:44:50 -0400 Subject: [PATCH 5/7] add ExtensionDType argument --- pandas/_testing/asserters.py | 5 +++-- pandas/core/arrays/base.py | 4 ++++ pandas/core/arrays/boolean.py | 9 +++++++-- pandas/core/arrays/categorical.py | 8 ++++++-- pandas/core/arrays/floating.py | 9 +++++++-- pandas/core/arrays/integer.py | 9 +++++++-- pandas/core/arrays/masked.py | 8 ++++++-- pandas/core/arrays/period.py | 4 +--- pandas/core/dtypes/dtypes.py | 3 ++- 9 files changed, 43 insertions(+), 16 deletions(-) diff --git a/pandas/_testing/asserters.py b/pandas/_testing/asserters.py index 527bf97bd4543..a29767153b021 100644 --- a/pandas/_testing/asserters.py +++ b/pandas/_testing/asserters.py @@ -398,8 +398,9 @@ def _get_ilevel_values(index, level): # skip exact index checking when `check_categorical` is False if check_exact and check_categorical: if not left.equals(right): - thesum = np.sum((left._values != right._values).astype(int)) - diff = thesum * 100.0 / len(left) + diff = ( + np.sum((left._values != right._values).astype(int)) * 100.0 / len(left) + ) msg = f"{obj} values are different ({np.round(diff, 5)} %)" raise_assert_detail(obj, msg, left, right) else: diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 54af0d3d98d5b..fe89ffd4617cd 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -520,6 +520,10 @@ def nbytes(self) -> int: def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: ... + @overload + def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: + ... + @overload def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: ... diff --git a/pandas/core/arrays/boolean.py b/pandas/core/arrays/boolean.py index c05040da1d49e..f357c922cf64f 100644 --- a/pandas/core/arrays/boolean.py +++ b/pandas/core/arrays/boolean.py @@ -37,6 +37,7 @@ from pandas.core.dtypes.missing import isna from pandas.core import ops +from pandas.core.arrays import ExtensionArray from pandas.core.arrays.masked import ( BaseMaskedArray, BaseMaskedDtype, @@ -397,11 +398,15 @@ def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value) @overload - def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: ... @overload - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: ... def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 5dde8b19a52bc..9cd73adb0e8c8 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -483,11 +483,15 @@ def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy=False): return Categorical(scalars, dtype=dtype, copy=copy) @overload - def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: ... @overload - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: ... def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: diff --git a/pandas/core/arrays/floating.py b/pandas/core/arrays/floating.py index 35dc8cec96cc3..364065873c19d 100644 --- a/pandas/core/arrays/floating.py +++ b/pandas/core/arrays/floating.py @@ -34,6 +34,7 @@ ) from pandas.core.dtypes.missing import isna +from pandas.core.arrays import ExtensionArray from pandas.core.arrays.numeric import ( NumericArray, NumericDtype, @@ -275,11 +276,15 @@ def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value, dtype=self.dtype) @overload - def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: ... @overload - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: ... def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index 0d3bcca5e3a49..4fe0aa65274d9 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -35,6 +35,7 @@ ) from pandas.core.dtypes.missing import isna +from pandas.core.arrays import ExtensionArray from pandas.core.arrays.masked import ( BaseMaskedArray, BaseMaskedDtype, @@ -336,11 +337,15 @@ def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value, dtype=self.dtype) @overload - def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: ... @overload - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: ... def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: diff --git a/pandas/core/arrays/masked.py b/pandas/core/arrays/masked.py index 52756a14601e4..ab7c1f8f9f540 100644 --- a/pandas/core/arrays/masked.py +++ b/pandas/core/arrays/masked.py @@ -301,11 +301,15 @@ def to_numpy( # type: ignore[override] return data @overload - def astype(self, dtype: NpDtype, copy: bool = True) -> np.ndarray: + def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: ... @overload - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: + ... + + @overload + def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: ... def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 471ee295ebd2f..04db06ee9fb66 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -341,9 +341,7 @@ def freq(self) -> BaseOffset: def __array__(self, dtype: NpDtype | None = None) -> np.ndarray: if dtype == "i8": return self.asi8 - # error: Non-overlapping equality check (left operand type: "Optional[Union[str, - # dtype[Any]]]", right operand type: "Type[bool]") - elif dtype == bool: # type: ignore[comparison-overlap] + elif dtype == bool: return ~self._isnan # This will raise TypeError for non-object dtypes diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py index 5dbe3def644a6..feb9eeefc9a3a 100644 --- a/pandas/core/dtypes/dtypes.py +++ b/pandas/core/dtypes/dtypes.py @@ -30,6 +30,7 @@ Dtype, DtypeObj, Ordered, + npt, type_t, ) @@ -1282,7 +1283,7 @@ class PandasDtype(ExtensionDtype): _metadata = ("_dtype",) - def __init__(self, dtype: str | np.dtype | PandasDtype | None): + def __init__(self, dtype: npt.DTypeLike | PandasDtype | None): if isinstance(dtype, PandasDtype): # make constructor univalent dtype = dtype.numpy_dtype From 80d780eedb81a720333d5789f1936b859872145c Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Mon, 5 Jul 2021 14:00:00 -0400 Subject: [PATCH 6/7] Use npt.DTypeLike --- pandas/_typing.py | 1 + pandas/core/arrays/base.py | 9 +++++---- pandas/core/arrays/boolean.py | 9 +++++---- pandas/core/arrays/categorical.py | 8 +++++--- pandas/core/arrays/floating.py | 10 +++++----- pandas/core/arrays/integer.py | 9 +++++---- pandas/core/arrays/masked.py | 9 +++++---- pandas/core/arrays/sparse/array.py | 3 ++- 8 files changed, 33 insertions(+), 25 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index a2ab40807757c..40603420692d4 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -124,6 +124,7 @@ # dtypes NpDtype = Union[str, np.dtype, type_t[Union[str, float, int, complex, bool, object]]] Dtype = Union["ExtensionDtype", NpDtype] +AstypeArg = Union["ExtensionDtype", npt.DTypeLike] # DtypeArg specifies all allowable dtypes in a functions its dtype argument DtypeArg = Union[Dtype, Dict[Hashable, Dtype]] DtypeObj = Union[np.dtype, "ExtensionDtype"] diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index fe89ffd4617cd..2a0f03a72e319 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -25,11 +25,12 @@ from pandas._libs import lib from pandas._typing import ( ArrayLike, + AstypeArg, Dtype, FillnaOptions, - NpDtype, PositionalIndexer, Shape, + npt, ) from pandas.compat import set_function_name from pandas.compat.numpy import function as nv @@ -517,7 +518,7 @@ def nbytes(self) -> int: # ------------------------------------------------------------------------ @overload - def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: + def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ... @overload @@ -525,10 +526,10 @@ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ... @overload - def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ... - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike: """ Cast to a NumPy array or ExtensionArray with 'dtype'. diff --git a/pandas/core/arrays/boolean.py b/pandas/core/arrays/boolean.py index f357c922cf64f..6f473249c9a7e 100644 --- a/pandas/core/arrays/boolean.py +++ b/pandas/core/arrays/boolean.py @@ -15,8 +15,9 @@ ) from pandas._typing import ( ArrayLike, + AstypeArg, Dtype, - NpDtype, + npt, type_t, ) from pandas.compat.numpy import function as nv @@ -398,7 +399,7 @@ def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value) @overload - def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: + def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ... @overload @@ -406,10 +407,10 @@ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ... @overload - def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ... - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike: """ Cast to a NumPy array or ExtensionArray with 'dtype'. diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 9cd73adb0e8c8..d1a1cfb35e3e7 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -33,11 +33,13 @@ from pandas._libs.lib import no_default from pandas._typing import ( ArrayLike, + AstypeArg, Dtype, NpDtype, Ordered, Scalar, Shape, + npt, type_t, ) from pandas.compat.numpy import function as nv @@ -483,7 +485,7 @@ def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy=False): return Categorical(scalars, dtype=dtype, copy=copy) @overload - def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: + def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ... @overload @@ -491,10 +493,10 @@ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ... @overload - def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ... - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike: """ Coerce this type to another dtype diff --git a/pandas/core/arrays/floating.py b/pandas/core/arrays/floating.py index 364065873c19d..25b4076bd23c6 100644 --- a/pandas/core/arrays/floating.py +++ b/pandas/core/arrays/floating.py @@ -11,9 +11,9 @@ ) from pandas._typing import ( ArrayLike, - Dtype, + AstypeArg, DtypeObj, - NpDtype, + npt, ) from pandas.compat.numpy import function as nv from pandas.util._decorators import cache_readonly @@ -276,7 +276,7 @@ def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value, dtype=self.dtype) @overload - def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: + def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ... @overload @@ -284,10 +284,10 @@ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ... @overload - def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ... - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike: """ Cast to a NumPy array or ExtensionArray with 'dtype'. diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index 4fe0aa65274d9..e62a2f95b0340 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -12,9 +12,10 @@ ) from pandas._typing import ( ArrayLike, + AstypeArg, Dtype, DtypeObj, - NpDtype, + npt, ) from pandas.compat.numpy import function as nv from pandas.util._decorators import cache_readonly @@ -337,7 +338,7 @@ def _coerce_to_array(self, value) -> tuple[np.ndarray, np.ndarray]: return coerce_to_array(value, dtype=self.dtype) @overload - def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: + def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ... @overload @@ -345,10 +346,10 @@ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ... @overload - def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ... - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike: """ Cast to a NumPy array or ExtensionArray with 'dtype'. diff --git a/pandas/core/arrays/masked.py b/pandas/core/arrays/masked.py index ab7c1f8f9f540..3a7a84004a493 100644 --- a/pandas/core/arrays/masked.py +++ b/pandas/core/arrays/masked.py @@ -16,10 +16,11 @@ ) from pandas._typing import ( ArrayLike, - Dtype, + AstypeArg, NpDtype, PositionalIndexer, Scalar, + npt, type_t, ) from pandas.errors import AbstractMethodError @@ -301,7 +302,7 @@ def to_numpy( # type: ignore[override] return data @overload - def astype(self, dtype: NpDtype, copy: bool = ...) -> np.ndarray: + def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ... @overload @@ -309,10 +310,10 @@ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ... @overload - def astype(self, dtype: Dtype, copy: bool = ...) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ... - def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike: + def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike: dtype = pandas_dtype(dtype) if is_dtype_equal(dtype, self.dtype): diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py index e642a8c603bc9..e55582a683daf 100644 --- a/pandas/core/arrays/sparse/array.py +++ b/pandas/core/arrays/sparse/array.py @@ -25,6 +25,7 @@ ) from pandas._libs.tslibs import NaT from pandas._typing import ( + AstypeArg, Dtype, NpDtype, Scalar, @@ -1056,7 +1057,7 @@ def _concat_same_type( return cls(data, sparse_index=sp_index, fill_value=fill_value) - def astype(self, dtype: Dtype | None = None, copy=True): + def astype(self, dtype: AstypeArg | None = None, copy=True): """ Change the dtype of a SparseArray. From 5ce17cbfd1bba64c0948dbc841b3c35c5b910d17 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Mon, 5 Jul 2021 14:17:25 -0400 Subject: [PATCH 7/7] quotes around npt.DTypeLike --- pandas/_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index 40603420692d4..1f0f7acf6b7d7 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -124,7 +124,7 @@ # dtypes NpDtype = Union[str, np.dtype, type_t[Union[str, float, int, complex, bool, object]]] Dtype = Union["ExtensionDtype", NpDtype] -AstypeArg = Union["ExtensionDtype", npt.DTypeLike] +AstypeArg = Union["ExtensionDtype", "npt.DTypeLike"] # DtypeArg specifies all allowable dtypes in a functions its dtype argument DtypeArg = Union[Dtype, Dict[Hashable, Dtype]] DtypeObj = Union[np.dtype, "ExtensionDtype"]