TYP: fix type-ignores in core (#40423)

Author: jbrockmendel

pandas/core/algorithms.py

@@ -1867,7 +1867,7 @@ def _sort_mixed(values):
     return np.concatenate([nums, np.asarray(strs, dtype=object)])
 
 
-def _sort_tuples(values: np.ndarray):
+def _sort_tuples(values: np.ndarray) -> np.ndarray:
     """
     Convert array of tuples (1d) to array or array (2d).
     We need to keep the columns separately as they contain different types and

pandas/core/arraylike.py

@@ -5,10 +5,7 @@
     ExtensionArray
 """
 import operator
-from typing import (
-    Any,
-    Callable,
-)
+from typing import Any
 import warnings
 
 import numpy as np
@@ -172,7 +169,7 @@ def _is_aligned(frame, other):
         return frame.columns.equals(other.index)
 
 
-def _maybe_fallback(ufunc: Callable, method: str, *inputs: Any, **kwargs: Any):
+def _maybe_fallback(ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any):
     """
     In the future DataFrame, inputs to ufuncs will be aligned before applying
     the ufunc, but for now we ignore the index but raise a warning if behaviour

pandas/core/arrays/boolean.py

@@ -331,7 +331,7 @@ def map_string(s):
 
     _HANDLED_TYPES = (np.ndarray, numbers.Number, bool, np.bool_)
 
-    def __array_ufunc__(self, ufunc, method: str, *inputs, **kwargs):
+    def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
         # For BooleanArray inputs, we apply the ufunc to ._data
         # and mask the result.
         if method == "reduce":

pandas/core/arrays/categorical.py

@@ -1388,7 +1388,7 @@ def __array__(self, dtype: Optional[NpDtype] = None) -> np.ndarray:
         # ndarray.
         return np.asarray(ret)
 
-    def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
+    def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
         # for binary ops, use our custom dunder methods
         result = ops.maybe_dispatch_ufunc_to_dunder_op(
             self, ufunc, method, *inputs, **kwargs
@@ -2429,7 +2429,7 @@ def replace(self, to_replace, value, inplace: bool = False):
 
     # ------------------------------------------------------------------------
     # String methods interface
-    def _str_map(self, f, na_value=np.nan, dtype=np.dtype(object)):
+    def _str_map(self, f, na_value=np.nan, dtype=np.dtype("object")):
         # Optimization to apply the callable `f` to the categories once
         # and rebuild the result by `take`ing from the result with the codes.
         # Returns the same type as the object-dtype implementation though.

pandas/core/arrays/numeric.py

@@ -152,7 +152,7 @@ def _arith_method(self, other, op):
 
     _HANDLED_TYPES = (np.ndarray, numbers.Number)
 
-    def __array_ufunc__(self, ufunc, method: str, *inputs, **kwargs):
+    def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
         # For NumericArray inputs, we apply the ufunc to ._data
         # and mask the result.
         if method == "reduce":

pandas/core/arrays/numpy_.py

@@ -137,7 +137,7 @@ def __array__(self, dtype: Optional[NpDtype] = None) -> np.ndarray:
 
     _HANDLED_TYPES = (np.ndarray, numbers.Number)
 
-    def __array_ufunc__(self, ufunc, method: str, *inputs, **kwargs):
+    def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
         # Lightly modified version of
         # https://numpy.org/doc/stable/reference/generated/numpy.lib.mixins.NDArrayOperatorsMixin.html
         # The primary modification is not boxing scalar return values

pandas/core/arrays/sparse/array.py

@@ -1396,7 +1396,7 @@ def mean(self, axis=0, *args, **kwargs):
 
     _HANDLED_TYPES = (np.ndarray, numbers.Number)
 
-    def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
+    def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
         out = kwargs.get("out", ())
 
         for x in inputs + out:

pandas/core/base.py

@@ -23,6 +23,7 @@
     Dtype,
     DtypeObj,
     IndexLabel,
+    Shape,
 )
 from pandas.compat import PYPY
 from pandas.compat.numpy import function as nv
@@ -389,7 +390,7 @@ def transpose(self: _T, *args, **kwargs) -> _T:
     )
 
     @property
-    def shape(self):
+    def shape(self) -> Shape:
         """
         Return a tuple of the shape of the underlying data.
         """
@@ -511,7 +512,7 @@ def to_numpy(
         copy: bool = False,
         na_value=lib.no_default,
         **kwargs,
-    ):
+    ) -> np.ndarray:
         """
         A NumPy ndarray representing the values in this Series or Index.
 
@@ -852,7 +853,7 @@ def __iter__(self):
             return map(self._values.item, range(self._values.size))
 
     @cache_readonly
-    def hasnans(self):
+    def hasnans(self) -> bool:
         """
         Return if I have any nans; enables various perf speedups.
         """

pandas/core/dtypes/cast.py

@@ -689,15 +689,14 @@ def _maybe_promote(dtype: np.dtype, fill_value=np.nan):
                 if fv.tz is None:
                     return dtype, fv.asm8
 
-        # error: Value of type variable "_DTypeScalar" of "dtype" cannot be "object"
-        return np.dtype(object), fill_value  # type: ignore[type-var]
+        return np.dtype("object"), fill_value
 
     elif issubclass(dtype.type, np.timedelta64):
         inferred, fv = infer_dtype_from_scalar(fill_value, pandas_dtype=True)
         if inferred == dtype:
             return dtype, fv
-        # error: Value of type variable "_DTypeScalar" of "dtype" cannot be "object"
-        return np.dtype(object), fill_value  # type: ignore[type-var]
+
+        return np.dtype("object"), fill_value
 
     elif is_float(fill_value):
         if issubclass(dtype.type, np.bool_):

pandas/core/dtypes/missing.py

@@ -648,8 +648,7 @@ def is_valid_na_for_dtype(obj, dtype: DtypeObj) -> bool:
         # Numeric
         return obj is not NaT and not isinstance(obj, (np.datetime64, np.timedelta64))
 
-    # error: Value of type variable "_DTypeScalar" of "dtype" cannot be "object"
-    elif dtype == np.dtype(object):  # type: ignore[type-var]
+    elif dtype == np.dtype("object"):
         # This is needed for Categorical, but is kind of weird
         return True
 

pandas/core/frame.py

@@ -123,6 +123,7 @@
     is_sequence,
     pandas_dtype,
 )
+from pandas.core.dtypes.dtypes import ExtensionDtype
 from pandas.core.dtypes.missing import (
     isna,
     notna,
@@ -584,25 +585,17 @@ def __init__(
             )
 
         elif isinstance(data, dict):
-            # error: Argument "dtype" to "dict_to_mgr" has incompatible type
-            # "Union[ExtensionDtype, str, dtype[Any], Type[object], None]"; expected
-            # "Union[dtype[Any], ExtensionDtype, None]"
-            mgr = dict_to_mgr(
-                data, index, columns, dtype=dtype, typ=manager  # type: ignore[arg-type]
-            )
+            mgr = dict_to_mgr(data, index, columns, dtype=dtype, typ=manager)
         elif isinstance(data, ma.MaskedArray):
             import numpy.ma.mrecords as mrecords
 
             # masked recarray
             if isinstance(data, mrecords.MaskedRecords):
-                # error: Argument 4 to "rec_array_to_mgr" has incompatible type
-                # "Union[ExtensionDtype, str, dtype[Any], Type[object], None]"; expected
-                # "Union[dtype[Any], ExtensionDtype, None]"
                 mgr = rec_array_to_mgr(
                     data,
                     index,
                     columns,
-                    dtype,  # type: ignore[arg-type]
+                    dtype,
                     copy,
                     typ=manager,
                 )
@@ -611,13 +604,10 @@ def __init__(
             else:
                 data = sanitize_masked_array(data)
                 mgr = ndarray_to_mgr(
-                    # error: Argument "dtype" to "ndarray_to_mgr" has incompatible type
-                    # "Union[ExtensionDtype, str, dtype[Any], Type[object], None]";
-                    # expected "Union[dtype[Any], ExtensionDtype, None]"
                     data,
                     index,
                     columns,
-                    dtype=dtype,  # type: ignore[arg-type]
+                    dtype=dtype,
                     copy=copy,
                     typ=manager,
                 )
@@ -626,14 +616,11 @@ def __init__(
             if data.dtype.names:
                 # i.e. numpy structured array
 
-                # error: Argument 4 to "rec_array_to_mgr" has incompatible type
-                # "Union[ExtensionDtype, str, dtype[Any], Type[object], None]"; expected
-                # "Union[dtype[Any], ExtensionDtype, None]"
                 mgr = rec_array_to_mgr(
                     data,
                     index,
                     columns,
-                    dtype,  # type: ignore[arg-type]
+                    dtype,
                     copy,
                     typ=manager,
                 )
@@ -642,24 +629,18 @@ def __init__(
                 mgr = dict_to_mgr(
                     # error: Item "ndarray" of "Union[ndarray, Series, Index]" has no
                     # attribute "name"
-                    # error: Argument "dtype" to "dict_to_mgr" has incompatible type
-                    # "Union[ExtensionDtype, str, dtype[Any], Type[object], None]";
-                    # expected "Union[dtype[Any], ExtensionDtype, None]"
                     {data.name: data},  # type: ignore[union-attr]
                     index,
                     columns,
-                    dtype=dtype,  # type: ignore[arg-type]
+                    dtype=dtype,
                     typ=manager,
                 )
             else:
                 mgr = ndarray_to_mgr(
-                    # error: Argument "dtype" to "ndarray_to_mgr" has incompatible type
-                    # "Union[ExtensionDtype, str, dtype[Any], Type[object], None]";
-                    # expected "Union[dtype[Any], ExtensionDtype, None]"
                     data,
                     index,
                     columns,
-                    dtype=dtype,  # type: ignore[arg-type]
+                    dtype=dtype,
                     copy=copy,
                     typ=manager,
                 )
@@ -680,46 +661,34 @@ def __init__(
                     arrays, columns, index = nested_data_to_arrays(
                         # error: Argument 3 to "nested_data_to_arrays" has incompatible
                         # type "Optional[Collection[Any]]"; expected "Optional[Index]"
-                        # error: Argument 4 to "nested_data_to_arrays" has incompatible
-                        # type "Union[ExtensionDtype, str, dtype[Any], Type[object],
-                        # None]"; expected "Union[dtype[Any], ExtensionDtype, None]"
                         data,
                         columns,
                         index,  # type: ignore[arg-type]
-                        dtype,  # type: ignore[arg-type]
+                        dtype,
                     )
                     mgr = arrays_to_mgr(
-                        # error: Argument "dtype" to "arrays_to_mgr" has incompatible
-                        # type "Union[ExtensionDtype, str, dtype[Any], Type[object],
-                        # None]"; expected "Union[dtype[Any], ExtensionDtype, None]"
                         arrays,
                         columns,
                         index,
                         columns,
-                        dtype=dtype,  # type: ignore[arg-type]
+                        dtype=dtype,
                         typ=manager,
                     )
                 else:
                     mgr = ndarray_to_mgr(
-                        # error: Argument "dtype" to "ndarray_to_mgr" has incompatible
-                        # type "Union[ExtensionDtype, str, dtype[Any], Type[object],
-                        # None]"; expected "Union[dtype[Any], ExtensionDtype, None]"
                         data,
                         index,
                         columns,
-                        dtype=dtype,  # type: ignore[arg-type]
+                        dtype=dtype,
                         copy=copy,
                         typ=manager,
                     )
             else:
-                # error: Argument "dtype" to "dict_to_mgr" has incompatible type
-                # "Union[ExtensionDtype, str, dtype[Any], Type[object], None]"; expected
-                # "Union[dtype[Any], ExtensionDtype, None]"
                 mgr = dict_to_mgr(
                     {},
                     index,
                     columns,
-                    dtype=dtype,  # type: ignore[arg-type]
+                    dtype=dtype,
                     typ=manager,
                 )
         # For data is scalar
@@ -731,16 +700,11 @@ def __init__(
                 dtype, _ = infer_dtype_from_scalar(data, pandas_dtype=True)
 
             # For data is a scalar extension dtype
-            if is_extension_array_dtype(dtype):
+            if isinstance(dtype, ExtensionDtype):
                 # TODO(EA2D): special case not needed with 2D EAs
 
                 values = [
-                    # error: Argument 3 to "construct_1d_arraylike_from_scalar"
-                    # has incompatible type "Union[ExtensionDtype, str, dtype,
-                    # Type[object]]"; expected "Union[dtype, ExtensionDtype]"
-                    construct_1d_arraylike_from_scalar(
-                        data, len(index), dtype  # type: ignore[arg-type]
-                    )
+                    construct_1d_arraylike_from_scalar(data, len(index), dtype)
                     for _ in range(len(columns))
                 ]
                 mgr = arrays_to_mgr(
@@ -750,13 +714,10 @@ def __init__(
                 # error: Incompatible types in assignment (expression has type
                 # "ndarray", variable has type "List[ExtensionArray]")
                 values = construct_2d_arraylike_from_scalar(  # type: ignore[assignment]
-                    # error: Argument 4 to "construct_2d_arraylike_from_scalar" has
-                    # incompatible type "Union[ExtensionDtype, str, dtype[Any],
-                    # Type[object]]"; expected "dtype[Any]"
                     data,
                     len(index),
                     len(columns),
-                    dtype,  # type: ignore[arg-type]
+                    dtype,
                     copy,
                 )
 

pandas/core/generic.py

@@ -44,6 +44,7 @@
     CompressionOptions,
     Dtype,
     DtypeArg,
+    DtypeObj,
     FilePathOrBuffer,
     FrameOrSeries,
     IndexKeyFunc,
@@ -411,7 +412,7 @@ def set_flags(
 
     @final
     @classmethod
-    def _validate_dtype(cls, dtype):
+    def _validate_dtype(cls, dtype) -> Optional[DtypeObj]:
         """ validate the passed dtype """
         if dtype is not None:
             dtype = pandas_dtype(dtype)
@@ -1995,13 +1996,9 @@ def __array_wrap__(
         )
 
     def __array_ufunc__(
-        self, ufunc: Callable, method: str, *inputs: Any, **kwargs: Any
+        self, ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any
     ):
-        # error: Argument 2 to "array_ufunc" has incompatible type "Callable[..., Any]";
-        # expected "ufunc"
-        return arraylike.array_ufunc(
-            self, ufunc, method, *inputs, **kwargs  # type: ignore[arg-type]
-        )
+        return arraylike.array_ufunc(self, ufunc, method, *inputs, **kwargs)
 
     # ideally we would define this to avoid the getattr checks, but
     # is slower
@@ -4900,7 +4897,6 @@ def _reindex_axes(
 
         return obj
 
-    @final
     def _needs_reindex_multi(self, axes, method, level) -> bool_t:
         """Check if we do need a multi reindex."""
         return (
@@ -6998,10 +6994,7 @@ def interpolate(
                     f"`limit_direction` must be 'backward' for method `{method}`"
                 )
 
-        # error: Value of type variable "_DTypeScalar" of "dtype" cannot be "object"
-        if obj.ndim == 2 and np.all(
-            obj.dtypes == np.dtype(object)  # type: ignore[type-var]
-        ):
+        if obj.ndim == 2 and np.all(obj.dtypes == np.dtype("object")):
             raise TypeError(
                 "Cannot interpolate with all object-dtype columns "
                 "in the DataFrame. Try setting at least one "
@@ -8488,15 +8481,12 @@ def ranker(data):
                 na_option=na_option,
                 pct=pct,
             )
-            # error: Incompatible types in assignment (expression has type
-            # "FrameOrSeries", variable has type "ndarray")
             # error: Argument 1 to "NDFrame" has incompatible type "ndarray"; expected
             # "Union[ArrayManager, BlockManager]"
-            ranks = self._constructor(  # type: ignore[assignment]
+            ranks_obj = self._constructor(
                 ranks, **data._construct_axes_dict()  # type: ignore[arg-type]
             )
-            # error: "ndarray" has no attribute "__finalize__"
-            return ranks.__finalize__(self, method="rank")  # type: ignore[attr-defined]
+            return ranks_obj.__finalize__(self, method="rank")
 
         # if numeric_only is None, and we can't get anything, we try with
         # numeric_only=True