From 66f08b951838b7be8624ed0b066dcbe34be07a79 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Thu, 19 May 2022 23:23:45 +0200 Subject: [PATCH 1/5] TYP: resolve mypy ignores in core/apply.py --- pandas/core/apply.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index c04d0821fffdc..199382deadfc3 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -32,6 +32,7 @@ AggObjType, Axis, NDFrameT, + npt, ) from pandas.util._decorators import cache_readonly from pandas.util._exceptions import find_stack_level @@ -584,18 +585,17 @@ def normalize_dictlike_arg( cols_sorted = list(safe_sort(list(cols))) raise KeyError(f"Column(s) {cols_sorted} do not exist") - is_aggregator = lambda x: isinstance(x, (list, tuple, dict)) + aggregator_types = (list, tuple, dict) # if we have a dict of any non-scalars # eg. {'A' : ['mean']}, normalize all to # be list-likes # Cannot use func.values() because arg may be a Series - if any(is_aggregator(x) for _, x in func.items()): + if any(isinstance(x, aggregator_types) for _, x in func.items()): new_func: AggFuncTypeDict = {} for k, v in func.items(): - if not is_aggregator(v): - # mypy can't realize v is not a list here - new_func[k] = [v] # type: ignore[list-item] + if not isinstance(v, aggregator_types): + new_func[k] = [v] else: new_func[k] = v func = new_func @@ -1079,6 +1079,7 @@ def apply(self) -> DataFrame | Series: # if we are a string, try to dispatch return self.apply_str() + # self.f is Callable return self.apply_standard() def agg(self): @@ -1116,7 +1117,8 @@ def apply_empty_result(self) -> Series: ) def apply_standard(self) -> DataFrame | Series: - f = self.f + # caller is responsible for ensuring that f is Callable + f = cast(Callable, self.f) obj = self.obj with np.errstate(all="ignore"): @@ -1129,14 +1131,9 @@ def apply_standard(self) -> DataFrame | Series: mapped = obj._values.map(f) else: values = obj.astype(object)._values - # error: Argument 2 to "map_infer" has incompatible type - # "Union[Callable[..., Any], str, List[Union[Callable[..., Any], str]], - # Dict[Hashable, Union[Union[Callable[..., Any], str], - # List[Union[Callable[..., Any], str]]]]]"; expected - # "Callable[[Any], Any]" mapped = lib.map_infer( values, - f, # type: ignore[arg-type] + f, convert=self.convert_dtype, ) @@ -1205,7 +1202,7 @@ def transform(self): def reconstruct_func( func: AggFuncType | None, **kwargs -) -> tuple[bool, AggFuncType | None, list[str] | None, list[int] | None]: +) -> tuple[bool, AggFuncType | None, list[str] | None, npt.NDArray[np.intp] | None]: """ This is the internal function to reconstruct func given if there is relabeling or not and also normalize the keyword to get new order of columns. @@ -1232,7 +1229,7 @@ def reconstruct_func( relabelling: bool, if there is relabelling or not func: normalized and mangled func columns: list of column names - order: list of columns indices + order: array of columns indices Examples -------- @@ -1244,7 +1241,7 @@ def reconstruct_func( """ relabeling = func is None and is_multi_agg_with_relabel(**kwargs) columns: list[str] | None = None - order: list[int] | None = None + order: npt.NDArray[np.intp] | None = None if not relabeling: if isinstance(func, list) and len(func) > len(set(func)): @@ -1291,7 +1288,9 @@ def is_multi_agg_with_relabel(**kwargs) -> bool: ) -def normalize_keyword_aggregation(kwargs: dict) -> tuple[dict, list[str], list[int]]: +def normalize_keyword_aggregation( + kwargs: dict, +) -> tuple[dict, list[str], npt.NDArray[np.intp]]: """ Normalize user-provided "named aggregation" kwargs. Transforms from the new ``Mapping[str, NamedAgg]`` style kwargs @@ -1345,9 +1344,7 @@ def normalize_keyword_aggregation(kwargs: dict) -> tuple[dict, list[str], list[i # get the new index of columns by comparison col_idx_order = Index(uniquified_aggspec).get_indexer(uniquified_order) - # error: Incompatible return value type (got "Tuple[defaultdict[Any, Any], - # Any, ndarray]", expected "Tuple[Dict[Any, Any], List[str], List[int]]") - return aggspec, columns, col_idx_order # type: ignore[return-value] + return aggspec, columns, col_idx_order def _make_unique_kwarg_list( From b21c771d77a9ae1bc0bc8414219f6a7a4da2cf1b Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 23 May 2022 20:42:54 +0200 Subject: [PATCH 2/5] TYP: narrow SeriesApply func type --- pandas/core/apply.py | 2 +- pandas/core/series.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 199382deadfc3..385f7f8589024 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -1049,7 +1049,7 @@ class SeriesApply(NDFrameApply): def __init__( self, obj: Series, - func: AggFuncType, + func: AggFuncTypeBase | list[AggFuncTypeBase], convert_dtype: bool, args, kwargs, diff --git a/pandas/core/series.py b/pandas/core/series.py index b740bac78b263..1249b1c48b578 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -32,7 +32,7 @@ ) from pandas._libs.lib import no_default from pandas._typing import ( - AggFuncType, + AggFuncTypeBase, ArrayLike, Axis, Dtype, @@ -4439,7 +4439,11 @@ def any( axis=_shared_doc_kwargs["axis"], ) def transform( - self, func: AggFuncType, axis: Axis = 0, *args, **kwargs + self, + func: AggFuncTypeBase | list[AggFuncTypeBase], + axis: Axis = 0, + *args, + **kwargs, ) -> DataFrame | Series: # Validate axis argument self._get_axis_number(axis) @@ -4450,7 +4454,7 @@ def transform( def apply( self, - func: AggFuncType, + func: AggFuncTypeBase | list[AggFuncTypeBase], convert_dtype: bool = True, args: tuple[Any, ...] = (), **kwargs, From 9d0de412c5094d5b05399c581169ef80e21b448d Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Mon, 23 May 2022 21:02:50 +0200 Subject: [PATCH 3/5] TYP: add SeriesAggFuncType alias --- pandas/_typing.py | 6 +++--- pandas/core/apply.py | 3 ++- pandas/core/series.py | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index e71859e91785e..81943544baebf 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -159,10 +159,10 @@ # types of `func` kwarg for DataFrame.aggregate and Series.aggregate AggFuncTypeBase = Union[Callable, str] -AggFuncTypeDict = Dict[Hashable, Union[AggFuncTypeBase, List[AggFuncTypeBase]]] +SeriesAggFuncType = Union[AggFuncTypeBase, List[AggFuncTypeBase]] +AggFuncTypeDict = Dict[Hashable, SeriesAggFuncType] AggFuncType = Union[ - AggFuncTypeBase, - List[AggFuncTypeBase], + SeriesAggFuncType, AggFuncTypeDict, ] AggObjType = Union[ diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 385f7f8589024..0672c9dfefbb1 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -32,6 +32,7 @@ AggObjType, Axis, NDFrameT, + SeriesAggFuncType, npt, ) from pandas.util._decorators import cache_readonly @@ -1049,7 +1050,7 @@ class SeriesApply(NDFrameApply): def __init__( self, obj: Series, - func: AggFuncTypeBase | list[AggFuncTypeBase], + func: SeriesAggFuncType, convert_dtype: bool, args, kwargs, diff --git a/pandas/core/series.py b/pandas/core/series.py index 1249b1c48b578..9df7c2aee5c2f 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -32,7 +32,6 @@ ) from pandas._libs.lib import no_default from pandas._typing import ( - AggFuncTypeBase, ArrayLike, Axis, Dtype, @@ -43,6 +42,7 @@ Level, NaPosition, Renamer, + SeriesAggFuncType, SingleManager, SortKind, StorageOptions, @@ -4440,7 +4440,7 @@ def any( ) def transform( self, - func: AggFuncTypeBase | list[AggFuncTypeBase], + func: SeriesAggFuncType, axis: Axis = 0, *args, **kwargs, @@ -4454,7 +4454,7 @@ def transform( def apply( self, - func: AggFuncTypeBase | list[AggFuncTypeBase], + func: SeriesAggFuncType, convert_dtype: bool = True, args: tuple[Any, ...] = (), **kwargs, From dc984dcddab67fe0a14f7bbe72fbb3010c927d43 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sun, 29 May 2022 15:00:53 +0200 Subject: [PATCH 4/5] Revert "TYP: add SeriesAggFuncType alias" This reverts commit 9d0de412c5094d5b05399c581169ef80e21b448d. --- pandas/_typing.py | 6 +++--- pandas/core/apply.py | 3 +-- pandas/core/series.py | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index 81943544baebf..e71859e91785e 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -159,10 +159,10 @@ # types of `func` kwarg for DataFrame.aggregate and Series.aggregate AggFuncTypeBase = Union[Callable, str] -SeriesAggFuncType = Union[AggFuncTypeBase, List[AggFuncTypeBase]] -AggFuncTypeDict = Dict[Hashable, SeriesAggFuncType] +AggFuncTypeDict = Dict[Hashable, Union[AggFuncTypeBase, List[AggFuncTypeBase]]] AggFuncType = Union[ - SeriesAggFuncType, + AggFuncTypeBase, + List[AggFuncTypeBase], AggFuncTypeDict, ] AggObjType = Union[ diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 0672c9dfefbb1..385f7f8589024 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -32,7 +32,6 @@ AggObjType, Axis, NDFrameT, - SeriesAggFuncType, npt, ) from pandas.util._decorators import cache_readonly @@ -1050,7 +1049,7 @@ class SeriesApply(NDFrameApply): def __init__( self, obj: Series, - func: SeriesAggFuncType, + func: AggFuncTypeBase | list[AggFuncTypeBase], convert_dtype: bool, args, kwargs, diff --git a/pandas/core/series.py b/pandas/core/series.py index 9df7c2aee5c2f..1249b1c48b578 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -32,6 +32,7 @@ ) from pandas._libs.lib import no_default from pandas._typing import ( + AggFuncTypeBase, ArrayLike, Axis, Dtype, @@ -42,7 +43,6 @@ Level, NaPosition, Renamer, - SeriesAggFuncType, SingleManager, SortKind, StorageOptions, @@ -4440,7 +4440,7 @@ def any( ) def transform( self, - func: SeriesAggFuncType, + func: AggFuncTypeBase | list[AggFuncTypeBase], axis: Axis = 0, *args, **kwargs, @@ -4454,7 +4454,7 @@ def transform( def apply( self, - func: SeriesAggFuncType, + func: AggFuncTypeBase | list[AggFuncTypeBase], convert_dtype: bool = True, args: tuple[Any, ...] = (), **kwargs, From 4d4360274babef1373db0f1f126ede051eecd334 Mon Sep 17 00:00:00 2001 From: Ilion Beyst Date: Sun, 29 May 2022 15:03:22 +0200 Subject: [PATCH 5/5] Revert "TYP: narrow SeriesApply func type" This reverts commit b21c771d77a9ae1bc0bc8414219f6a7a4da2cf1b. --- pandas/core/apply.py | 2 +- pandas/core/series.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 385f7f8589024..199382deadfc3 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -1049,7 +1049,7 @@ class SeriesApply(NDFrameApply): def __init__( self, obj: Series, - func: AggFuncTypeBase | list[AggFuncTypeBase], + func: AggFuncType, convert_dtype: bool, args, kwargs, diff --git a/pandas/core/series.py b/pandas/core/series.py index 1249b1c48b578..b740bac78b263 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -32,7 +32,7 @@ ) from pandas._libs.lib import no_default from pandas._typing import ( - AggFuncTypeBase, + AggFuncType, ArrayLike, Axis, Dtype, @@ -4439,11 +4439,7 @@ def any( axis=_shared_doc_kwargs["axis"], ) def transform( - self, - func: AggFuncTypeBase | list[AggFuncTypeBase], - axis: Axis = 0, - *args, - **kwargs, + self, func: AggFuncType, axis: Axis = 0, *args, **kwargs ) -> DataFrame | Series: # Validate axis argument self._get_axis_number(axis) @@ -4454,7 +4450,7 @@ def transform( def apply( self, - func: AggFuncTypeBase | list[AggFuncTypeBase], + func: AggFuncType, convert_dtype: bool = True, args: tuple[Any, ...] = (), **kwargs,