From 322065880f7646667809762a9aa6fdd9c0d3e5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 18 Mar 2022 22:51:52 -0400 Subject: [PATCH 1/5] TYP: rename (TypeVar approach) --- pandas/_typing.py | 5 ++- pandas/core/frame.py | 56 +++++++++++++++++++++++++++--- pandas/core/generic.py | 23 ++++++++++--- pandas/core/groupby/groupby.py | 3 +- pandas/core/series.py | 63 ++++++++++++++++++++++++++++++---- pandas/io/json/_normalize.py | 9 +++-- 6 files changed, 135 insertions(+), 24 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index 30244e025e430..f7d707d7531bb 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -141,7 +141,10 @@ ] # For functions like rename that convert one label to another -Renamer = Union[Mapping[Hashable, Any], Callable[[Hashable], Hashable]] +HashableT = TypeVar("HashableT", bound=Hashable) +HashableTa = TypeVar("HashableTa", bound=Hashable) +HashableTb = TypeVar("HashableTb", bound=Hashable) +Renamer = Union[Mapping[HashableT, Hashable], Callable[[HashableT], Hashable]] # to maintain type information across generic functions and parametrization T = TypeVar("T") diff --git a/pandas/core/frame.py b/pandas/core/frame.py index d56e4ef451954..daef9ff17df10 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -58,6 +58,9 @@ FloatFormatType, FormattersType, Frequency, + HashableT, + HashableTa, + HashableTb, IgnoreRaise, IndexKeyFunc, IndexLabel, @@ -5030,17 +5033,62 @@ def drop( # type: ignore[override] errors=errors, ) + @overload + def rename( + self, + mapper: Renamer[HashableT] | None = ..., + *, + index: Renamer[HashableTa] | None = ..., + columns: Renamer[HashableTb] | None = ..., + axis: Axis | None = ..., + copy: bool = ..., + inplace: Literal[True], + level: Level | None = ..., + errors: IgnoreRaise = ..., + ) -> None: + ... + + @overload + def rename( + self, + mapper: Renamer[HashableT] | None = ..., + *, + index: Renamer[HashableTa] | None = ..., + columns: Renamer[HashableTb] | None = ..., + axis: Axis | None = ..., + copy: bool = ..., + inplace: Literal[False] = ..., + level: Level | None = ..., + errors: IgnoreRaise = ..., + ) -> DataFrame: + ... + + @overload def rename( self, - mapper: Renamer | None = None, + mapper: Renamer[HashableT] | None = ..., *, - index: Renamer | None = None, - columns: Renamer | None = None, + index: Renamer[HashableTa] | None = ..., + columns: Renamer[HashableTb] | None = ..., + axis: Axis | None = ..., + copy: bool = ..., + inplace: bool = ..., + level: Level | None = ..., + errors: IgnoreRaise = ..., + ) -> DataFrame | None: + ... + + def rename( + self, + mapper: Renamer[HashableT] | None = None, + *, + index: Renamer[HashableTa] | None = None, + columns: Renamer[HashableTb] | None = None, axis: Axis | None = None, copy: bool = True, inplace: bool = False, level: Level | None = None, - errors: str = "ignore", + errors: IgnoreRaise = "ignore", ) -> DataFrame | None: """ Alter axes labels. diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 700a8f6a39f8d..04dd69ecc4177 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -44,6 +44,9 @@ DtypeArg, DtypeObj, FilePath, + HashableT, + HashableTa, + HashableTb, IgnoreRaise, IndexKeyFunc, IndexLabel, @@ -974,10 +977,10 @@ def squeeze(self, axis=None): def _rename( self: NDFrameT, - mapper: Renamer | None = None, + mapper: Renamer[HashableT] | None = None, *, - index: Renamer | None = None, - columns: Renamer | None = None, + index: Renamer[HashableTa] | None = None, + columns: Renamer[HashableTb] | None = None, axis: Axis | None = None, copy: bool_t = True, inplace: bool_t = False, @@ -1110,9 +1113,19 @@ def _rename( else: # use the mapper argument if axis and self._get_axis_number(axis) == 1: - columns = mapper + # error: Incompatible types in assignment (expression has type + # "Optional[Union[Mapping[HashableT, Hashable], Callable[ + # [HashableT], Hashable]]]", variable has type "Optional[Union[ + # Mapping[HashableTb, Hashable], Callable[[HashableTb], Hashable + # ]]]") + columns = mapper # type: ignore[assignment] else: - index = mapper + # error: Incompatible types in assignment (expression has type + # "Optional[Union[Mapping[HashableT, Hashable], Callable[[ + # HashableT], Hashable]]]", variable has type "Optional[Union[ + # Mapping[HashableTa, Hashable], Callable[[HashableTa], Hashable + # ]]]") + index = mapper # type: ignore[assignment] self._check_inplace_and_allows_duplicate_labels(inplace) result = self if inplace else self.copy(deep=copy) diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py index 91c35d7555705..f649cce985474 100644 --- a/pandas/core/groupby/groupby.py +++ b/pandas/core/groupby/groupby.py @@ -2204,8 +2204,7 @@ def size(self) -> DataFrame | Series: result = self._obj_1d_constructor(result) if not self.as_index: - # Item "None" of "Optional[Series]" has no attribute "reset_index" - result = result.rename("size").reset_index() # type: ignore[union-attr] + result = result.rename("size").reset_index() return self._reindex_output(result, fill_value=0) diff --git a/pandas/core/series.py b/pandas/core/series.py index 83c5e8206952c..341b5fc5125c7 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -38,10 +38,12 @@ Dtype, DtypeObj, FillnaOptions, + HashableT, IgnoreRaise, IndexKeyFunc, Level, NaPosition, + Renamer, SingleManager, SortKind, StorageOptions, @@ -4617,15 +4619,54 @@ def align( broadcast_axis=broadcast_axis, ) + @overload def rename( self, - index=None, + index: Renamer[HashableT] | Hashable | None = ..., *, - axis=None, - copy=True, - inplace=False, - level=None, - errors="ignore", + axis: Axis | None = ..., + copy: bool = ..., + inplace: Literal[True], + level: Level | None = ..., + errors: IgnoreRaise = ..., + ) -> None: + ... + + @overload + def rename( + self, + index: Renamer[HashableT] | Hashable | None = ..., + *, + axis: Axis | None = ..., + copy: bool = ..., + inplace: Literal[False] = ..., + level: Level | None = ..., + errors: IgnoreRaise = ..., + ) -> Series: + ... + + @overload + def rename( + self, + index: Renamer[HashableT] | Hashable | None = ..., + *, + axis: Axis | None = ..., + copy: bool = ..., + inplace: bool = ..., + level: Level | None = ..., + errors: IgnoreRaise = ..., + ) -> Series | None: + ... + + def rename( + self, + index: Renamer[HashableT] | Hashable | None = None, + *, + axis: Axis | None = None, + copy: bool = True, + inplace: bool = False, + level: Level | None = None, + errors: IgnoreRaise = "ignore", ) -> Series | None: """ Alter Series index labels or name. @@ -4691,8 +4732,16 @@ def rename( axis = self._get_axis_number(axis) if callable(index) or is_dict_like(index): + # error: Argument 1 to "_rename" of "NDFrame" has incompatible type + # "Union[Union[Mapping[Hashable, Any], Callable[[Hashable], Hashable]], + # Hashable, None]"; expected "Union[Mapping[Hashable, Any], + # Callable[[Hashable], Hashable], None]" return super()._rename( - index, copy=copy, inplace=inplace, level=level, errors=errors + index, # type: ignore[arg-type] + copy=copy, + inplace=inplace, + level=level, + errors=errors, ) else: return self._set_name(index, inplace=inplace) diff --git a/pandas/io/json/_normalize.py b/pandas/io/json/_normalize.py index 4a2e49fd85f45..35dc162e4ef4a 100644 --- a/pandas/io/json/_normalize.py +++ b/pandas/io/json/_normalize.py @@ -9,7 +9,9 @@ import copy from typing import ( Any, + Callable, DefaultDict, + Hashable, Iterable, ) @@ -520,11 +522,8 @@ def _recursive_extract(data, path, seen_meta, level=0): result = DataFrame(records) if record_prefix is not None: - # Incompatible types in assignment (expression has type "Optional[DataFrame]", - # variable has type "DataFrame") - result = result.rename( # type: ignore[assignment] - columns=lambda x: f"{record_prefix}{x}" - ) + rename_fun: Callable[[Hashable], str] = lambda x: f"{record_prefix}{x}" + result = result.rename(columns=rename_fun) # Data types, a problem for k, v in meta_vals.items(): From 17a705b9a9690acbba7fbdc3f96b699d2bfc35fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Thu, 31 Mar 2022 15:56:55 -0400 Subject: [PATCH 2/5] use Any --- pandas/_typing.py | 5 +---- pandas/core/frame.py | 27 ++++++++++++--------------- pandas/core/generic.py | 23 +++++------------------ pandas/core/series.py | 9 ++++----- pandas/io/json/_normalize.py | 3 +-- 5 files changed, 23 insertions(+), 44 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index f7d707d7531bb..35c057df43322 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -141,10 +141,7 @@ ] # For functions like rename that convert one label to another -HashableT = TypeVar("HashableT", bound=Hashable) -HashableTa = TypeVar("HashableTa", bound=Hashable) -HashableTb = TypeVar("HashableTb", bound=Hashable) -Renamer = Union[Mapping[HashableT, Hashable], Callable[[HashableT], Hashable]] +Renamer = Union[Mapping[Any, Hashable], Callable[[Any], Hashable]] # to maintain type information across generic functions and parametrization T = TypeVar("T") diff --git a/pandas/core/frame.py b/pandas/core/frame.py index daef9ff17df10..6d4deb79b4898 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -58,9 +58,6 @@ FloatFormatType, FormattersType, Frequency, - HashableT, - HashableTa, - HashableTb, IgnoreRaise, IndexKeyFunc, IndexLabel, @@ -5036,10 +5033,10 @@ def drop( # type: ignore[override] @overload def rename( self, - mapper: Renamer[HashableT] | None = ..., + mapper: Renamer | None = ..., *, - index: Renamer[HashableTa] | None = ..., - columns: Renamer[HashableTb] | None = ..., + index: Renamer | None = ..., + columns: Renamer | None = ..., axis: Axis | None = ..., copy: bool = ..., inplace: Literal[True], @@ -5051,10 +5048,10 @@ def rename( @overload def rename( self, - mapper: Renamer[HashableT] | None = ..., + mapper: Renamer | None = ..., *, - index: Renamer[HashableTa] | None = ..., - columns: Renamer[HashableTb] | None = ..., + index: Renamer | None = ..., + columns: Renamer | None = ..., axis: Axis | None = ..., copy: bool = ..., inplace: Literal[False] = ..., @@ -5066,10 +5063,10 @@ def rename( @overload def rename( self, - mapper: Renamer[HashableT] | None = ..., + mapper: Renamer | None = ..., *, - index: Renamer[HashableTa] | None = ..., - columns: Renamer[HashableTb] | None = ..., + index: Renamer | None = ..., + columns: Renamer | None = ..., axis: Axis | None = ..., copy: bool = ..., inplace: bool = ..., @@ -5080,10 +5077,10 @@ def rename( def rename( self, - mapper: Renamer[HashableT] | None = None, + mapper: Renamer | None = None, *, - index: Renamer[HashableTa] | None = None, - columns: Renamer[HashableTb] | None = None, + index: Renamer | None = None, + columns: Renamer | None = None, axis: Axis | None = None, copy: bool = True, inplace: bool = False, diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 04dd69ecc4177..700a8f6a39f8d 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -44,9 +44,6 @@ DtypeArg, DtypeObj, FilePath, - HashableT, - HashableTa, - HashableTb, IgnoreRaise, IndexKeyFunc, IndexLabel, @@ -977,10 +974,10 @@ def squeeze(self, axis=None): def _rename( self: NDFrameT, - mapper: Renamer[HashableT] | None = None, + mapper: Renamer | None = None, *, - index: Renamer[HashableTa] | None = None, - columns: Renamer[HashableTb] | None = None, + index: Renamer | None = None, + columns: Renamer | None = None, axis: Axis | None = None, copy: bool_t = True, inplace: bool_t = False, @@ -1113,19 +1110,9 @@ def _rename( else: # use the mapper argument if axis and self._get_axis_number(axis) == 1: - # error: Incompatible types in assignment (expression has type - # "Optional[Union[Mapping[HashableT, Hashable], Callable[ - # [HashableT], Hashable]]]", variable has type "Optional[Union[ - # Mapping[HashableTb, Hashable], Callable[[HashableTb], Hashable - # ]]]") - columns = mapper # type: ignore[assignment] + columns = mapper else: - # error: Incompatible types in assignment (expression has type - # "Optional[Union[Mapping[HashableT, Hashable], Callable[[ - # HashableT], Hashable]]]", variable has type "Optional[Union[ - # Mapping[HashableTa, Hashable], Callable[[HashableTa], Hashable - # ]]]") - index = mapper # type: ignore[assignment] + index = mapper self._check_inplace_and_allows_duplicate_labels(inplace) result = self if inplace else self.copy(deep=copy) diff --git a/pandas/core/series.py b/pandas/core/series.py index 341b5fc5125c7..6d522d5163d48 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -38,7 +38,6 @@ Dtype, DtypeObj, FillnaOptions, - HashableT, IgnoreRaise, IndexKeyFunc, Level, @@ -4622,7 +4621,7 @@ def align( @overload def rename( self, - index: Renamer[HashableT] | Hashable | None = ..., + index: Renamer | Hashable | None = ..., *, axis: Axis | None = ..., copy: bool = ..., @@ -4635,7 +4634,7 @@ def rename( @overload def rename( self, - index: Renamer[HashableT] | Hashable | None = ..., + index: Renamer | Hashable | None = ..., *, axis: Axis | None = ..., copy: bool = ..., @@ -4648,7 +4647,7 @@ def rename( @overload def rename( self, - index: Renamer[HashableT] | Hashable | None = ..., + index: Renamer | Hashable | None = ..., *, axis: Axis | None = ..., copy: bool = ..., @@ -4660,7 +4659,7 @@ def rename( def rename( self, - index: Renamer[HashableT] | Hashable | None = None, + index: Renamer | Hashable | None = None, *, axis: Axis | None = None, copy: bool = True, diff --git a/pandas/io/json/_normalize.py b/pandas/io/json/_normalize.py index 35dc162e4ef4a..ec45ed65908b8 100644 --- a/pandas/io/json/_normalize.py +++ b/pandas/io/json/_normalize.py @@ -522,8 +522,7 @@ def _recursive_extract(data, path, seen_meta, level=0): result = DataFrame(records) if record_prefix is not None: - rename_fun: Callable[[Hashable], str] = lambda x: f"{record_prefix}{x}" - result = result.rename(columns=rename_fun) + result = result.rename(columns=lambda x: f"{record_prefix}{x}") # Data types, a problem for k, v in meta_vals.items(): From 4dda358ca976aa067fa39dc2682bc663c923fc81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Thu, 31 Mar 2022 16:05:54 -0400 Subject: [PATCH 3/5] flake8 --- pandas/io/json/_normalize.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/io/json/_normalize.py b/pandas/io/json/_normalize.py index ec45ed65908b8..e77d60d2d4950 100644 --- a/pandas/io/json/_normalize.py +++ b/pandas/io/json/_normalize.py @@ -9,9 +9,7 @@ import copy from typing import ( Any, - Callable, DefaultDict, - Hashable, Iterable, ) From 3314b914ee22859188e69d6af23e0c4c4c7fd8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 1 Apr 2022 08:10:49 -0400 Subject: [PATCH 4/5] update ignore comment --- pandas/core/series.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/core/series.py b/pandas/core/series.py index 6d522d5163d48..1d3509cac0edd 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -4731,10 +4731,10 @@ def rename( axis = self._get_axis_number(axis) if callable(index) or is_dict_like(index): - # error: Argument 1 to "_rename" of "NDFrame" has incompatible type - # "Union[Union[Mapping[Hashable, Any], Callable[[Hashable], Hashable]], - # Hashable, None]"; expected "Union[Mapping[Hashable, Any], - # Callable[[Hashable], Hashable], None]" + # error: Argument 1 to "_rename" of "NDFrame" has incompatible + # type "Union[Union[Mapping[Any, Hashable], Callable[[Any], + # Hashable]], Hashable, None]"; expected "Union[Mapping[Any, + # Hashable], Callable[[Any], Hashable], None]" return super()._rename( index, # type: ignore[arg-type] copy=copy, From f8682f220f5e798503e05ec964159a138b7e83e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 1 Apr 2022 14:16:48 -0400 Subject: [PATCH 5/5] remove doc-string of NDFrame._rename --- pandas/core/generic.py | 111 +---------------------------------------- 1 file changed, 1 insertion(+), 110 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 700a8f6a39f8d..2e65e3139ffa1 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -984,117 +984,8 @@ def _rename( level: Level | None = None, errors: str = "ignore", ) -> NDFrameT | None: - """ - Alter axes input function or functions. Function / dict values must be - unique (1-to-1). Labels not contained in a dict / Series will be left - as-is. Extra labels listed don't throw an error. Alternatively, change - ``Series.name`` with a scalar value (Series only). - - Parameters - ---------- - %(axes)s : scalar, list-like, dict-like or function, optional - Scalar or list-like will alter the ``Series.name`` attribute, - and raise on DataFrame. - dict-like or functions are transformations to apply to - that axis' values - copy : bool, default True - Also copy underlying data. - inplace : bool, default False - Whether to return a new {klass}. If True then value of copy is - ignored. - level : int or level name, default None - In case of a MultiIndex, only rename labels in the specified - level. - errors : {'ignore', 'raise'}, default 'ignore' - If 'raise', raise a `KeyError` when a dict-like `mapper`, `index`, - or `columns` contains labels that are not present in the Index - being transformed. - If 'ignore', existing keys will be renamed and extra keys will be - ignored. - - Returns - ------- - renamed : {klass} (new object) - - Raises - ------ - KeyError - If any of the labels is not found in the selected axis and - "errors='raise'". + # called by Series.rename and DataFrame.rename - See Also - -------- - NDFrame.rename_axis - - Examples - -------- - >>> s = pd.Series([1, 2, 3]) - >>> s - 0 1 - 1 2 - 2 3 - dtype: int64 - >>> s.rename("my_name") # scalar, changes Series.name - 0 1 - 1 2 - 2 3 - Name: my_name, dtype: int64 - >>> s.rename(lambda x: x ** 2) # function, changes labels - 0 1 - 1 2 - 4 3 - dtype: int64 - >>> s.rename({1: 3, 2: 5}) # mapping, changes labels - 0 1 - 3 2 - 5 3 - dtype: int64 - - Since ``DataFrame`` doesn't have a ``.name`` attribute, - only mapping-type arguments are allowed. - - >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]}) - >>> df.rename(2) - Traceback (most recent call last): - ... - TypeError: 'int' object is not callable - - ``DataFrame.rename`` supports two calling conventions - - * ``(index=index_mapper, columns=columns_mapper, ...)`` - * ``(mapper, axis={'index', 'columns'}, ...)`` - - We *highly* recommend using keyword arguments to clarify your - intent. - - >>> df.rename(index=str, columns={"A": "a", "B": "c"}) - a c - 0 1 4 - 1 2 5 - 2 3 6 - - >>> df.rename(index=str, columns={"A": "a", "C": "c"}) - a B - 0 1 4 - 1 2 5 - 2 3 6 - - Using axis-style parameters - - >>> df.rename(str.lower, axis='columns') - a b - 0 1 4 - 1 2 5 - 2 3 6 - - >>> df.rename({1: 2, 2: 4}, axis='index') - A B - 0 1 4 - 2 2 5 - 4 3 6 - - See the :ref:`user guide ` for more. - """ if mapper is None and index is None and columns is None: raise TypeError("must pass an index to rename")