From 193ff2f02f516dfdec807b20cb8fa14b9ff29f64 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Fri, 12 Mar 2021 11:28:52 +0000 Subject: [PATCH 1/6] more overloads for DatetimeLikeArrayMixin.view --- pandas/core/arrays/datetimelike.py | 10 ++++++++++ pandas/core/arrays/period.py | 6 +----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index bd5cc04659a06..e060720525df0 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -123,6 +123,8 @@ from pandas.tseries import frequencies if TYPE_CHECKING: + from typing import Literal + from pandas.core.arrays import ( DatetimeArray, TimedeltaArray, @@ -458,6 +460,14 @@ def astype(self, dtype, copy=True): def view(self: DatetimeLikeArrayT) -> DatetimeLikeArrayT: ... + @overload + def view(self, dtype: Literal["M8[ns]"]) -> DatetimeArray: + ... + + @overload + def view(self, dtype: Literal["m8[ns]"]) -> TimedeltaArray: + ... + @overload def view(self, dtype: Optional[Dtype] = ...) -> ArrayLike: ... diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index a39182d61a8fb..d91522a9e1bb6 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -643,11 +643,7 @@ def fillna(self, value=None, method=None, limit=None) -> PeriodArray: if method is not None: # view as dt64 so we get treated as timelike in core.missing dta = self.view("M8[ns]") - # error: Item "ndarray" of "Union[ExtensionArray, ndarray]" has no attribute - # "fillna" - result = dta.fillna( # type: ignore[union-attr] - value=value, method=method, limit=limit - ) + result = dta.fillna(value=value, method=method, limit=limit) return result.view(self.dtype) return super().fillna(value=value, method=method, limit=limit) From a6500c79803acb7bae2ed2f858363da32fd39f8c Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Fri, 12 Mar 2021 12:00:07 +0000 Subject: [PATCH 2/6] overload astype_nansafe --- pandas/core/arrays/sparse/dtype.py | 9 ++++----- pandas/core/dtypes/cast.py | 15 +++++++++++++++ pandas/io/parsers/base_parser.py | 5 +---- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pandas/core/arrays/sparse/dtype.py b/pandas/core/arrays/sparse/dtype.py index 8e55eb5f3d358..f780bea5e4124 100644 --- a/pandas/core/arrays/sparse/dtype.py +++ b/pandas/core/arrays/sparse/dtype.py @@ -9,6 +9,7 @@ Optional, Tuple, Type, + cast, ) import warnings @@ -342,11 +343,9 @@ def update_dtype(self, dtype): if is_extension_array_dtype(dtype): raise TypeError("sparse arrays of extension dtypes not supported") - # error: Item "ExtensionArray" of "Union[ExtensionArray, ndarray]" has no - # attribute "item" - fill_value = astype_nansafe( # type: ignore[union-attr] - np.array(self.fill_value), dtype - ).item() + dtype = cast(np.dtype, dtype) + + fill_value = astype_nansafe(np.array(self.fill_value), dtype).item() dtype = cls(dtype, fill_value=fill_value) return dtype diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 44650500e0f65..d97407fcd00c4 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -25,6 +25,7 @@ Type, Union, cast, + overload, ) import warnings @@ -1164,6 +1165,20 @@ def astype_td64_unit_conversion( return result +@overload +def astype_nansafe( + arr: np.ndarray, dtype: np.dtype, copy: bool = ..., skipna: bool = ... +) -> np.ndarray: + ... + + +@overload +def astype_nansafe( + arr: np.ndarray, dtype: ExtensionDtype, copy: bool = ..., skipna: bool = ... +) -> ExtensionArray: + ... + + def astype_nansafe( arr: np.ndarray, dtype: DtypeObj, copy: bool = True, skipna: bool = False ) -> ArrayLike: diff --git a/pandas/io/parsers/base_parser.py b/pandas/io/parsers/base_parser.py index 8cfbae3cafc18..2e08a9eee7089 100644 --- a/pandas/io/parsers/base_parser.py +++ b/pandas/io/parsers/base_parser.py @@ -724,10 +724,7 @@ def _cast_types(self, values, cast_type, column): # TODO: this is for consistency with # c-parser which parses all categories # as strings - - # error: Argument 2 to "astype_nansafe" has incompatible type - # "Type[str]"; expected "Union[dtype[Any], ExtensionDtype]" - values = astype_nansafe(values, str) # type: ignore[arg-type] + values = astype_nansafe(values, np.string_) cats = Index(values).unique().dropna() values = Categorical._from_inferred_categories( From 1570db3da7851520edd3a0e69a07a9c63ae16622 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Fri, 12 Mar 2021 12:15:26 +0000 Subject: [PATCH 3/6] Literal for numpy order argument --- pandas/core/dtypes/cast.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index d97407fcd00c4..ce91276bc6cf4 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -108,6 +108,8 @@ ) if TYPE_CHECKING: + from typing import Literal + from pandas import Series from pandas.core.arrays import ( DatetimeArray, @@ -1205,14 +1207,10 @@ def astype_nansafe( flags = arr.flags flat = arr.ravel("K") result = astype_nansafe(flat, dtype, copy=copy, skipna=skipna) - order = "F" if flags.f_contiguous else "C" + order: Literal["C", "F"] = "F" if flags.f_contiguous else "C" # error: Item "ExtensionArray" of "Union[ExtensionArray, ndarray]" has no # attribute "reshape" - # error: No overload variant of "reshape" of "_ArrayOrScalarCommon" matches - # argument types "Tuple[int, ...]", "str" - return result.reshape( # type: ignore[union-attr,call-overload] - arr.shape, order=order - ) + return result.reshape(arr.shape, order=order) # type: ignore[union-attr] # We get here with 0-dim from sparse arr = np.atleast_1d(arr) From c5374e37feb90fdd6169e572a3a32d8b5601e64a Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Fri, 12 Mar 2021 12:41:13 +0000 Subject: [PATCH 4/6] np.string_ -> np.dtype(str) --- pandas/io/parsers/base_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/parsers/base_parser.py b/pandas/io/parsers/base_parser.py index 2e08a9eee7089..1abb1e13ea2c7 100644 --- a/pandas/io/parsers/base_parser.py +++ b/pandas/io/parsers/base_parser.py @@ -724,7 +724,7 @@ def _cast_types(self, values, cast_type, column): # TODO: this is for consistency with # c-parser which parses all categories # as strings - values = astype_nansafe(values, np.string_) + values = astype_nansafe(values, np.dtype(str)) cats = Index(values).unique().dropna() values = Categorical._from_inferred_categories( From c8740e5482d2a778484d97f8e097352c719526b7 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 13 Mar 2021 13:44:13 +0000 Subject: [PATCH 5/6] post merge clean --- pandas/io/parsers/base_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/io/parsers/base_parser.py b/pandas/io/parsers/base_parser.py index 1abb1e13ea2c7..a011a789bf17c 100644 --- a/pandas/io/parsers/base_parser.py +++ b/pandas/io/parsers/base_parser.py @@ -724,6 +724,7 @@ def _cast_types(self, values, cast_type, column): # TODO: this is for consistency with # c-parser which parses all categories # as strings + values = astype_nansafe(values, np.dtype(str)) cats = Index(values).unique().dropna() From 683d441698cb1b99dd03d004c86e5220fb6177d4 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 13 Mar 2021 13:57:00 +0000 Subject: [PATCH 6/6] remove cast --- pandas/core/arrays/sparse/dtype.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/core/arrays/sparse/dtype.py b/pandas/core/arrays/sparse/dtype.py index f780bea5e4124..d2d05577d14df 100644 --- a/pandas/core/arrays/sparse/dtype.py +++ b/pandas/core/arrays/sparse/dtype.py @@ -9,7 +9,6 @@ Optional, Tuple, Type, - cast, ) import warnings @@ -28,7 +27,6 @@ from pandas.core.dtypes.cast import astype_nansafe from pandas.core.dtypes.common import ( is_bool_dtype, - is_extension_array_dtype, is_object_dtype, is_scalar, is_string_dtype, @@ -340,11 +338,9 @@ def update_dtype(self, dtype): dtype = pandas_dtype(dtype) if not isinstance(dtype, cls): - if is_extension_array_dtype(dtype): + if not isinstance(dtype, np.dtype): raise TypeError("sparse arrays of extension dtypes not supported") - dtype = cast(np.dtype, dtype) - fill_value = astype_nansafe(np.array(self.fill_value), dtype).item() dtype = cls(dtype, fill_value=fill_value)