TYP: Update mypy and pyright (#56493)

* changes for mypy

* changes for pyright

* 1.1.400 would require too many difficult changes

Author: twoertwein

.pre-commit-config.yaml

@@ -132,11 +132,11 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].318
+        - [email protected].339
     -   id: pyright
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues
-        entry: pyright --skipunannotated -p pyright_reportGeneralTypeIssues.json --level warning
+        entry: pyright -p pyright_reportGeneralTypeIssues.json --level warning
         language: node
         pass_filenames: false
         types: [python]

doc/source/whatsnew/v2.2.0.rst

@@ -346,6 +346,8 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | Package         | Minimum Version | Changed |
 +=================+=================+=========+
+| mypy (dev)      | 1.7.1           |    X    |
++-----------------+-----------------+---------+
 |                 |                 |    X    |
 +-----------------+-----------------+---------+
 

environment.yml

@@ -76,7 +76,7 @@ dependencies:
 
   # code checks
   - flake8=6.1.0  # run in subprocess over docstring examples
-  - mypy=1.4.1  # pre-commit uses locally installed mypy
+  - mypy=1.7.1  # pre-commit uses locally installed mypy
   - tokenize-rt  # scripts/check_for_inconsistent_pandas_namespace.py
   - pre-commit>=3.6.0
 

pandas/_config/config.py

@@ -220,6 +220,8 @@ def get_default_val(pat: str):
 class DictWrapper:
     """provide attribute-style access to a nested dict"""
 
+    d: dict[str, Any]
+
     def __init__(self, d: dict[str, Any], prefix: str = "") -> None:
         object.__setattr__(self, "d", d)
         object.__setattr__(self, "prefix", prefix)
@@ -250,7 +252,7 @@ def __getattr__(self, key: str):
         else:
             return _get_option(prefix)
 
-    def __dir__(self) -> Iterable[str]:
+    def __dir__(self) -> list[str]:
         return list(self.d.keys())
 
 

pandas/_libs/tslibs/nattype.pyi

@@ -8,6 +8,7 @@ import typing
 import numpy as np
 
 from pandas._libs.tslibs.period import Period
+from pandas._typing import Self
 
 NaT: NaTType
 iNaT: int
@@ -132,4 +133,9 @@ class NaTType:
     __le__: _NatComparison
     __gt__: _NatComparison
     __ge__: _NatComparison
+    def __sub__(self, other: Self | timedelta | datetime) -> Self: ...
+    def __rsub__(self, other: Self | timedelta | datetime) -> Self: ...
+    def __add__(self, other: Self | timedelta | datetime) -> Self: ...
+    def __radd__(self, other: Self | timedelta | datetime) -> Self: ...
+    def __hash__(self) -> int: ...
     def as_unit(self, unit: str, round_ok: bool = ...) -> NaTType: ...

pandas/_libs/tslibs/timestamps.pyi

@@ -10,7 +10,6 @@ from typing import (
     ClassVar,
     Literal,
     TypeAlias,
-    TypeVar,
     overload,
 )
 
@@ -28,7 +27,6 @@ from pandas._typing import (
     TimestampNonexistent,
 )
 
-_DatetimeT = TypeVar("_DatetimeT", bound=datetime)
 _TimeZones: TypeAlias = str | _tzinfo | None | int
 
 def integer_op_not_supported(obj: object) -> TypeError: ...
@@ -42,7 +40,7 @@ class Timestamp(datetime):
     _value: int  # np.int64
     # error: "__new__" must return a class instance (got "Union[Timestamp, NaTType]")
     def __new__(  # type: ignore[misc]
-        cls: type[_DatetimeT],
+        cls: type[Self],
         ts_input: np.integer | float | str | _date | datetime | np.datetime64 = ...,
         year: int | None = ...,
         month: int | None = ...,
@@ -57,7 +55,7 @@ class Timestamp(datetime):
         tz: _TimeZones = ...,
         unit: str | int | None = ...,
         fold: int | None = ...,
-    ) -> _DatetimeT | NaTType: ...
+    ) -> Self | NaTType: ...
     @classmethod
     def _from_value_and_reso(
         cls, value: int, reso: int, tz: _TimeZones

pandas/_testing/_hypothesis.py

@@ -54,8 +54,12 @@
     DATETIME_NO_TZ = st.datetimes()
 
 DATETIME_JAN_1_1900_OPTIONAL_TZ = st.datetimes(
-    min_value=pd.Timestamp(1900, 1, 1).to_pydatetime(),
-    max_value=pd.Timestamp(1900, 1, 1).to_pydatetime(),
+    min_value=pd.Timestamp(
+        1900, 1, 1
+    ).to_pydatetime(),  # pyright: ignore[reportGeneralTypeIssues]
+    max_value=pd.Timestamp(
+        1900, 1, 1
+    ).to_pydatetime(),  # pyright: ignore[reportGeneralTypeIssues]
     timezones=st.one_of(st.none(), dateutil_timezones(), pytz_timezones()),
 )
 

pandas/_typing.py

@@ -234,7 +234,9 @@ def __reversed__(self) -> Iterator[_T_co]:
 
 # types of `func` kwarg for DataFrame.aggregate and Series.aggregate
 AggFuncTypeBase = Union[Callable, str]
-AggFuncTypeDict = dict[Hashable, Union[AggFuncTypeBase, list[AggFuncTypeBase]]]
+AggFuncTypeDict = MutableMapping[
+    Hashable, Union[AggFuncTypeBase, list[AggFuncTypeBase]]
+]
 AggFuncType = Union[
     AggFuncTypeBase,
     list[AggFuncTypeBase],

pandas/conftest.py

@@ -1353,7 +1353,7 @@ def fixed_now_ts() -> Timestamp:
     """
     Fixture emits fixed Timestamp.now()
     """
-    return Timestamp(
+    return Timestamp(  # pyright: ignore[reportGeneralTypeIssues]
         year=2021, month=1, day=1, hour=12, minute=4, second=13, microsecond=22
     )
 

pandas/core/_numba/kernels/var_.py

@@ -116,7 +116,7 @@ def sliding_var(
                     ssqdm_x,
                     compensation_add,
                     num_consecutive_same_value,
-                    prev_value,  # pyright: ignore[reportGeneralTypeIssues]
+                    prev_value,
                 )
         else:
             for j in range(start[i - 1], s):
@@ -141,7 +141,7 @@ def sliding_var(
                     ssqdm_x,
                     compensation_add,
                     num_consecutive_same_value,
-                    prev_value,  # pyright: ignore[reportGeneralTypeIssues]
+                    prev_value,
                 )
 
         if nobs >= min_periods and nobs > ddof:

pandas/core/apply.py

@@ -9,7 +9,6 @@
     TYPE_CHECKING,
     Any,
     Callable,
-    DefaultDict,
     Literal,
     cast,
 )
@@ -63,6 +62,7 @@
         Generator,
         Hashable,
         Iterable,
+        MutableMapping,
         Sequence,
     )
 
@@ -1642,7 +1642,7 @@ def transform(self):
 
 def reconstruct_func(
     func: AggFuncType | None, **kwargs
-) -> tuple[bool, AggFuncType, list[str] | None, npt.NDArray[np.intp] | None]:
+) -> tuple[bool, AggFuncType, tuple[str, ...] | None, npt.NDArray[np.intp] | None]:
     """
     This is the internal function to reconstruct func given if there is relabeling
     or not and also normalize the keyword to get new order of columns.
@@ -1668,7 +1668,7 @@ def reconstruct_func(
     -------
     relabelling: bool, if there is relabelling or not
     func: normalized and mangled func
-    columns: list of column names
+    columns: tuple of column names
     order: array of columns indices
 
     Examples
@@ -1680,7 +1680,7 @@ def reconstruct_func(
     (False, 'min', None, None)
     """
     relabeling = func is None and is_multi_agg_with_relabel(**kwargs)
-    columns: list[str] | None = None
+    columns: tuple[str, ...] | None = None
     order: npt.NDArray[np.intp] | None = None
 
     if not relabeling:
@@ -1696,7 +1696,14 @@ def reconstruct_func(
             raise TypeError("Must provide 'func' or tuples of '(column, aggfunc).")
 
     if relabeling:
-        func, columns, order = normalize_keyword_aggregation(kwargs)
+        # error: Incompatible types in assignment (expression has type
+        # "MutableMapping[Hashable, list[Callable[..., Any] | str]]", variable has type
+        # "Callable[..., Any] | str | list[Callable[..., Any] | str] |
+        # MutableMapping[Hashable, Callable[..., Any] | str | list[Callable[..., Any] |
+        # str]] | None")
+        func, columns, order = normalize_keyword_aggregation(  # type: ignore[assignment]
+            kwargs
+        )
     assert func is not None
 
     return relabeling, func, columns, order
@@ -1730,7 +1737,11 @@ def is_multi_agg_with_relabel(**kwargs) -> bool:
 
 def normalize_keyword_aggregation(
     kwargs: dict,
-) -> tuple[dict, list[str], npt.NDArray[np.intp]]:
+) -> tuple[
+    MutableMapping[Hashable, list[AggFuncTypeBase]],
+    tuple[str, ...],
+    npt.NDArray[np.intp],
+]:
     """
     Normalize user-provided "named aggregation" kwargs.
     Transforms from the new ``Mapping[str, NamedAgg]`` style kwargs
@@ -1744,7 +1755,7 @@ def normalize_keyword_aggregation(
     -------
     aggspec : dict
         The transformed kwargs.
-    columns : List[str]
+    columns : tuple[str, ...]
         The user-provided keys.
     col_idx_order : List[int]
         List of columns indices.
@@ -1759,9 +1770,7 @@ def normalize_keyword_aggregation(
     # Normalize the aggregation functions as Mapping[column, List[func]],
     # process normally, then fixup the names.
     # TODO: aggspec type: typing.Dict[str, List[AggScalar]]
-    # May be hitting https://github.com/python/mypy/issues/5958
-    # saying it doesn't have an attribute __name__
-    aggspec: DefaultDict = defaultdict(list)
+    aggspec = defaultdict(list)
     order = []
     columns, pairs = list(zip(*kwargs.items()))
 

pandas/core/arrays/_ranges.py

@@ -57,9 +57,7 @@ def generate_regular_range(
     b: int | np.int64 | np.uint64
     e: int | np.int64 | np.uint64
     try:
-        td = td.as_unit(  # pyright: ignore[reportGeneralTypeIssues]
-            unit, round_ok=False
-        )
+        td = td.as_unit(unit, round_ok=False)
     except ValueError as err:
         raise ValueError(
             f"freq={freq} is incompatible with unit={unit}. "

pandas/core/arrays/categorical.py

@@ -2894,9 +2894,7 @@ def _validate(data):
         if not isinstance(data.dtype, CategoricalDtype):
             raise AttributeError("Can only use .cat accessor with a 'category' dtype")
 
-    # error: Signature of "_delegate_property_get" incompatible with supertype
-    # "PandasDelegate"
-    def _delegate_property_get(self, name: str):  # type: ignore[override]
+    def _delegate_property_get(self, name: str):
         return getattr(self._parent, name)
 
     # error: Signature of "_delegate_property_set" incompatible with supertype

pandas/core/arrays/datetimelike.py

@@ -600,7 +600,9 @@ def _validate_scalar(
             raise TypeError(msg)
 
         elif isinstance(value, self._recognized_scalars):
-            value = self._scalar_type(value)
+            # error: Argument 1 to "Timestamp" has incompatible type "object"; expected
+            # "integer[Any] | float | str | date | datetime | datetime64"
+            value = self._scalar_type(value)  # type: ignore[arg-type]
 
         else:
             msg = self._validation_error_message(value, allow_listlike)

pandas/core/arrays/datetimes.py

@@ -397,10 +397,8 @@ def _from_sequence_not_strict(
         result._maybe_pin_freq(freq, validate_kwds)
         return result
 
-    # error: Signature of "_generate_range" incompatible with supertype
-    # "DatetimeLikeArrayMixin"
     @classmethod
-    def _generate_range(  # type: ignore[override]
+    def _generate_range(
         cls,
         start,
         end,

pandas/core/arrays/timedeltas.py

@@ -267,10 +267,8 @@ def _from_sequence_not_strict(
         result._maybe_pin_freq(freq, {})
         return result
 
-    # Signature of "_generate_range" incompatible with supertype
-    # "DatetimeLikeArrayMixin"
     @classmethod
-    def _generate_range(  # type: ignore[override]
+    def _generate_range(
         cls, start, end, periods, freq, closed=None, *, unit: str | None = None
     ) -> Self:
         periods = dtl.validate_periods(periods)

pandas/core/base.py

@@ -1310,12 +1310,10 @@ def factorize(
     # This overload is needed so that the call to searchsorted in
     # pandas.core.resample.TimeGrouper._get_period_bins picks the correct result
 
-    @overload
-    # The following ignore is also present in numpy/__init__.pyi
-    # Possibly a mypy bug??
     # error: Overloaded function signatures 1 and 2 overlap with incompatible
-    # return types  [misc]
-    def searchsorted(  # type: ignore[misc]
+    # return types
+    @overload
+    def searchsorted(  # type: ignore[overload-overlap]
         self,
         value: ScalarLike_co,
         side: Literal["left", "right"] = ...,

pandas/core/frame.py

@@ -6896,14 +6896,7 @@ def f(vals) -> tuple[np.ndarray, int]:
             vals = (col.values for name, col in self.items() if name in subset)
             labels, shape = map(list, zip(*map(f, vals)))
 
-            ids = get_group_index(
-                labels,
-                # error: Argument 1 to "tuple" has incompatible type "List[_T]";
-                # expected "Iterable[int]"
-                tuple(shape),  # type: ignore[arg-type]
-                sort=False,
-                xnull=False,
-            )
+            ids = get_group_index(labels, tuple(shape), sort=False, xnull=False)
             result = self._constructor_sliced(duplicated(ids, keep), index=self.index)
         return result.__finalize__(self, method="duplicated")
 

pandas/core/generic.py

@@ -8005,7 +8005,9 @@ def replace(
             if items:
                 keys, values = zip(*items)
             else:
-                keys, values = ([], [])
+                # error: Incompatible types in assignment (expression has type
+                # "list[Never]", variable has type "tuple[Any, ...]")
+                keys, values = ([], [])  # type: ignore[assignment]
 
             are_mappings = [is_dict_like(v) for v in values]
 
@@ -8020,7 +8022,12 @@ def replace(
                 value_dict = {}
 
                 for k, v in items:
-                    keys, values = list(zip(*v.items())) or ([], [])
+                    # error: Incompatible types in assignment (expression has type
+                    # "list[Never]", variable has type "tuple[Any, ...]")
+                    keys, values = list(zip(*v.items())) or (  # type: ignore[assignment]
+                        [],
+                        [],
+                    )
 
                     to_rep_dict[k] = list(keys)
                     value_dict[k] = list(values)

pandas/core/indexes/accessors.py

@@ -85,9 +85,7 @@ def _get_values(self):
             f"cannot convert an object of type {type(data)} to a datetimelike index"
         )
 
-    # error: Signature of "_delegate_property_get" incompatible with supertype
-    # "PandasDelegate"
-    def _delegate_property_get(self, name: str):  # type: ignore[override]
+    def _delegate_property_get(self, name: str):
         from pandas import Series
 
         values = self._get_values()
@@ -175,7 +173,7 @@ def __init__(self, data: Series, orig) -> None:
         self._orig = orig
         self._freeze()
 
-    def _delegate_property_get(self, name: str):  # type: ignore[override]
+    def _delegate_property_get(self, name: str):
         if not hasattr(self._parent.array, f"_dt_{name}"):
             raise NotImplementedError(
                 f"dt.{name} is not supported for {self._parent.dtype}"