From 05dcd231bd80e0b06443ce6f1944134e70b96c81 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 29 Jul 2019 20:33:46 +0100 Subject: [PATCH 01/11] TYPING: add some type hints to core.generic --- environment.yml | 1 + pandas/_typing.py | 1 + pandas/core/generic.py | 344 +++++++++++++++++-------- pandas/core/series.py | 3 +- pandas/io/formats/format.py | 4 +- pandas/tests/indexing/test_indexing.py | 12 +- requirements-dev.txt | 1 + 7 files changed, 239 insertions(+), 127 deletions(-) diff --git a/environment.yml b/environment.yml index 93e8302b498a0..4deeeff81df59 100644 --- a/environment.yml +++ b/environment.yml @@ -24,6 +24,7 @@ dependencies: - isort # check that imports are in the right order - mypy - pycodestyle # used by flake8 + - typing_extensions # documentation - gitpython # obtain contributors from git for whatsnew diff --git a/pandas/_typing.py b/pandas/_typing.py index 837a7a89e0b83..df8367cc1ab89 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -28,3 +28,4 @@ Scalar = Union[str, int, float] Axis = Union[str, int] Ordered = Optional[bool] +Level = Union[str, int] diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 97a0b04146297..2291fe31b9352 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -6,7 +6,21 @@ import operator import pickle from textwrap import dedent -from typing import Callable, Dict, FrozenSet, List, Optional, Set +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + FrozenSet, + Iterable, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, +) import warnings import weakref @@ -49,7 +63,7 @@ from pandas.core.dtypes.missing import isna, notna import pandas as pd -from pandas._typing import Dtype +from pandas._typing import Axis, Dtype, Level from pandas.core import missing, nanops import pandas.core.algorithms as algos from pandas.core.base import PandasObject, SelectionMixin @@ -71,6 +85,18 @@ from pandas.io.formats.printing import pprint_thing from pandas.tseries.frequencies import to_offset +if TYPE_CHECKING: + from pandas import Series, DataFrame + from typing_extensions import Literal + Errors = Literal["ignore", "raise"] + How = Literal["first", "last"] +else: + Errors = str + How = str + +bool_t = bool +FrameOrSeries = TypeVar("FrameOrSeries", bound="NDFrame") + # goal is to be able to define the docs close to function, while still being # able to share _shared_docs = dict() # type: Dict[str, str] @@ -155,6 +181,11 @@ class NDFrame(PandasObject, SelectionMixin): _metadata = [] # type: List[str] _is_copy = None _data = None # type: BlockManager + _AXIS_ALIASES = None # type: Dict[str, int] + _AXIS_NAMES = None # type: Dict[int, str] + _AXIS_NUMBERS = None # type: Dict[str, int] + _AXIS_REVERSED = None + _AXIS_LEN = None # type: int # ---------------------------------------------------------------------- # Constructors @@ -182,7 +213,9 @@ def __init__( object.__setattr__(self, "_data", data) object.__setattr__(self, "_item_cache", {}) - def _init_mgr(self, mgr, axes=None, dtype=None, copy=False): + def _init_mgr( + self, mgr: BlockManager, axes: Dict[str, Any], dtype=None, copy: bool = False + ) -> BlockManager: """ passed a manager and a axes dict """ for a, axe in axes.items(): if axe is not None: @@ -243,7 +276,7 @@ def _validate_dtype(self, dtype): # Construction @property - def _constructor(self): + def _constructor(self: FrameOrSeries) -> FrameOrSeries: """Used when a manipulation result has the same dimensions as the original. """ @@ -330,21 +363,27 @@ def set_axis(a, i): assert not isinstance(ns, dict) - def _construct_axes_dict(self, axes=None, **kwargs): + def _construct_axes_dict( + self, axes: Optional[Iterable[str]] = None, **kwargs + ) -> Dict[str, Index]: """Return an axes dictionary for myself.""" d = {a: self._get_axis(a) for a in (axes or self._AXIS_ORDERS)} d.update(kwargs) return d @staticmethod - def _construct_axes_dict_from(self, axes, **kwargs): + def _construct_axes_dict_from(self, axes, **kwargs) -> Dict[str, Index]: """Return an axes dictionary for the passed axes.""" d = {a: ax for a, ax in zip(self._AXIS_ORDERS, axes)} d.update(kwargs) return d def _construct_axes_from_arguments( - self, args, kwargs, require_all=False, sentinel=None + self, + args, + kwargs: Dict[str, Any], + require_all: bool = False, + sentinel: Optional[object] = None, ): """Construct and returns axes if supplied in args/kwargs. @@ -386,7 +425,9 @@ def _construct_axes_from_arguments( return axes, kwargs @classmethod - def _from_axes(cls, data, axes, **kwargs): + def _from_axes( + cls: Type[FrameOrSeries], data: BlockManager, axes, **kwargs + ) -> FrameOrSeries: # for construction from BlockManager if isinstance(data, BlockManager): return cls(data, **kwargs) @@ -398,7 +439,7 @@ def _from_axes(cls, data, axes, **kwargs): return cls(data, **d) @classmethod - def _get_axis_number(cls, axis): + def _get_axis_number(cls, axis) -> int: axis = cls._AXIS_ALIASES.get(axis, axis) if is_integer(axis): if axis in cls._AXIS_NAMES: @@ -411,7 +452,7 @@ def _get_axis_number(cls, axis): raise ValueError("No axis named {0} for object type {1}".format(axis, cls)) @classmethod - def _get_axis_name(cls, axis): + def _get_axis_name(cls, axis) -> str: axis = cls._AXIS_ALIASES.get(axis, axis) if isinstance(axis, str): if axis in cls._AXIS_NUMBERS: @@ -423,12 +464,12 @@ def _get_axis_name(cls, axis): pass raise ValueError("No axis named {0} for object type {1}".format(axis, cls)) - def _get_axis(self, axis): + def _get_axis(self, axis: Axis) -> Index: name = self._get_axis_name(axis) return getattr(self, name) @classmethod - def _get_block_manager_axis(cls, axis): + def _get_block_manager_axis(cls, axis: Axis) -> int: """Map the axis to the block_manager axis.""" axis = cls._get_axis_number(axis) if cls._AXIS_REVERSED: @@ -436,7 +477,7 @@ def _get_block_manager_axis(cls, axis): return m - axis return axis - def _get_axis_resolvers(self, axis): + def _get_axis_resolvers(self, axis: str) -> Dict[str, Union["Series", MultiIndex]]: # index or columns axis_index = getattr(self, axis) d = dict() @@ -466,13 +507,15 @@ def _get_axis_resolvers(self, axis): d[axis] = dindex return d - def _get_index_resolvers(self): + def _get_index_resolvers(self) -> Dict[str, Union["Series", MultiIndex]]: d = {} for axis_name in self._AXIS_ORDERS: d.update(self._get_axis_resolvers(axis_name)) return d - def _get_space_character_free_column_resolvers(self): + def _get_space_character_free_column_resolvers( + self + ) -> Dict[Union[int, str], "Series"]: """Return the space character free column resolvers of a dataframe. Column names with spaces are 'cleaned up' so that they can be referred @@ -484,7 +527,7 @@ def _get_space_character_free_column_resolvers(self): return {_remove_spaces_column_name(k): v for k, v in self.items()} @property - def _info_axis(self): + def _info_axis(self) -> Index: return getattr(self, self._info_axis_name) @property @@ -555,12 +598,12 @@ def size(self): return np.prod(self.shape) @property - def _selected_obj(self): + def _selected_obj(self: FrameOrSeries) -> FrameOrSeries: """ internal compat with SelectionMixin """ return self @property - def _obj_with_exclusions(self): + def _obj_with_exclusions(self: FrameOrSeries) -> FrameOrSeries: """ internal compat with SelectionMixin """ return self @@ -663,7 +706,7 @@ def set_axis(self, labels, axis=0, inplace=False): obj.set_axis(labels, axis=axis, inplace=True) return obj - def _set_axis(self, axis, labels): + def _set_axis(self, axis: int, labels) -> None: self._data.set_axis(axis, labels) self._clear_item_cache() @@ -1318,7 +1361,12 @@ class name if not inplace: return result - def _set_axis_name(self, name, axis=0, inplace=False): + def _set_axis_name( + self: FrameOrSeries, + name: Optional[Union[List[str], str]], + axis: Axis = 0, + inplace: bool = False, + ) -> Optional[FrameOrSeries]: """ Set the name(s) of the axis. @@ -1378,11 +1426,12 @@ def _set_axis_name(self, name, axis=0, inplace=False): renamed.set_axis(idx, axis=axis, inplace=True) if not inplace: return renamed + return None # ---------------------------------------------------------------------- # Comparison Methods - def _indexed_same(self, other): + def _indexed_same(self: FrameOrSeries, other: FrameOrSeries) -> bool: return all( self._get_axis(a).equals(other._get_axis(a)) for a in self._AXIS_ORDERS ) @@ -1476,7 +1525,7 @@ def equals(self, other): # ------------------------------------------------------------------------- # Unary Methods - def __neg__(self): + def __neg__(self: FrameOrSeries) -> FrameOrSeries: values = com.values_from_object(self) if is_bool_dtype(values): arr = operator.inv(values) @@ -1492,7 +1541,7 @@ def __neg__(self): ) return self.__array_wrap__(arr) - def __pos__(self): + def __pos__(self: FrameOrSeries) -> FrameOrSeries: values = com.values_from_object(self) if is_bool_dtype(values) or is_period_arraylike(values): arr = values @@ -1508,7 +1557,7 @@ def __pos__(self): ) return self.__array_wrap__(arr) - def __invert__(self): + def __invert__(self: FrameOrSeries) -> FrameOrSeries: try: arr = operator.inv(com.values_from_object(self)) return self.__array_wrap__(arr) @@ -1554,10 +1603,10 @@ def bool(self): self.__nonzero__() - def __abs__(self): + def __abs__(self: FrameOrSeries) -> FrameOrSeries: return self.abs() - def __round__(self, decimals=0): + def __round__(self: FrameOrSeries, decimals: int = 0) -> FrameOrSeries: return self.round(decimals) # ------------------------------------------------------------------------- @@ -1568,7 +1617,7 @@ def __round__(self, decimals=0): # operations should utilize/extend these methods when possible so that we # have consistent precedence and validation logic throughout the library. - def _is_level_reference(self, key, axis=0): + def _is_level_reference(self, key, axis: Axis = 0) -> bool_t: """ Test whether a key is a level reference for a given axis. @@ -1598,7 +1647,7 @@ def _is_level_reference(self, key, axis=0): and not self._is_label_reference(key, axis=axis) ) - def _is_label_reference(self, key, axis=0): + def _is_label_reference(self, key, axis: Axis = 0) -> bool_t: """ Test whether a key is a label reference for a given axis. @@ -1627,7 +1676,7 @@ def _is_label_reference(self, key, axis=0): and any(key in self.axes[ax] for ax in other_axes) ) - def _is_label_or_level_reference(self, key, axis=0): + def _is_label_or_level_reference(self, key: str, axis: Axis = 0) -> bool_t: """ Test whether a key is a label or level reference for a given axis. @@ -1651,7 +1700,7 @@ def _is_label_or_level_reference(self, key, axis=0): key, axis=axis ) - def _check_label_or_level_ambiguity(self, key, axis=0): + def _check_label_or_level_ambiguity(self, key, axis: Axis = 0) -> None: """ Check whether `key` is ambiguous. @@ -1700,7 +1749,7 @@ def _check_label_or_level_ambiguity(self, key, axis=0): ) raise ValueError(msg) - def _get_label_or_level_values(self, key, axis=0): + def _get_label_or_level_values(self, key, axis: Axis = 0): """ Return a 1-D array of values associated with `key`, a label or level from the given `axis`. @@ -1772,7 +1821,9 @@ def _get_label_or_level_values(self, key, axis=0): return values - def _drop_labels_or_levels(self, keys, axis=0): + def _drop_labels_or_levels( + self: FrameOrSeries, keys: Union[List[str], str], axis: Axis = 0 + ) -> FrameOrSeries: """ Drop labels and/or levels for the given `axis`. @@ -1903,7 +1954,7 @@ def __len__(self): """Returns length of info axis""" return len(self._info_axis) - def __contains__(self, key): + def __contains__(self, key) -> bool_t: """True if the key is in the info axis""" return key in self._info_axis @@ -1966,7 +2017,7 @@ def empty(self): def __array__(self, dtype=None): return com.values_from_object(self) - def __array_wrap__(self, result, context=None): + def __array_wrap__(self: FrameOrSeries, result, context=None) -> FrameOrSeries: d = self._construct_axes_dict(self._AXIS_ORDERS, copy=False) return self._constructor(result, **d).__finalize__(self) @@ -2001,11 +2052,11 @@ def to_dense(self): # ---------------------------------------------------------------------- # Picklability - def __getstate__(self): + def __getstate__(self) -> Dict[str, Any]: meta = {k: getattr(self, k, None) for k in self._metadata} return dict(_data=self._data, _typ=self._typ, _metadata=self._metadata, **meta) - def __setstate__(self, state): + def __setstate__(self, state) -> None: if isinstance(state, BlockManager): self._data = state @@ -2042,7 +2093,7 @@ def __setstate__(self, state): # old pickling format, for compatibility self._unpickle_matrix_compat(state) - self._item_cache = {} + self._item_cache = {} # type: Dict # ---------------------------------------------------------------------- # Rendering Methods @@ -2053,7 +2104,7 @@ def __repr__(self): prepr = "[%s]" % ",".join(map(pprint_thing, self)) return "%s(%s)" % (self.__class__.__name__, prepr) - def _repr_latex_(self): + def _repr_latex_(self) -> Optional[str]: """ Returns a LaTeX representation for a particular object. Mainly for use with nbconvert (jupyter notebook conversion to pdf). @@ -2063,7 +2114,7 @@ def _repr_latex_(self): else: return None - def _repr_data_resource_(self): + def _repr_data_resource_(self) -> Optional[Dict]: """ Not a real Jupyter special repr method, but we use the same naming convention. @@ -2074,6 +2125,7 @@ def _repr_data_resource_(self): data.to_json(orient="table"), object_pairs_hook=collections.OrderedDict ) return payload + return None # ---------------------------------------------------------------------- # I/O Methods @@ -3211,35 +3263,37 @@ def _create_indexer(cls, name, indexer): # ---------------------------------------------------------------------- # Lookup Caching - def _set_as_cached(self, item, cacher): + def _set_as_cached(self, item, cacher) -> None: """Set the _cacher attribute on the calling object with a weakref to cacher. """ self._cacher = (item, weakref.ref(cacher)) - def _reset_cacher(self): + def _reset_cacher(self) -> None: """Reset the cacher.""" if hasattr(self, "_cacher"): del self._cacher - def _maybe_cache_changed(self, item, value): + def _maybe_cache_changed(self, item: Union[str, int], value) -> None: """The object has called back to us saying maybe it has changed. """ self._data.set(item, value) @property - def _is_cached(self): + def _is_cached(self) -> bool_t: """Return boolean indicating if self is cached or not.""" return getattr(self, "_cacher", None) is not None - def _get_cacher(self): + def _get_cacher(self: FrameOrSeries) -> Optional[FrameOrSeries]: """return my cacher or None""" cacher = getattr(self, "_cacher", None) if cacher is not None: cacher = cacher[1]() return cacher - def _maybe_update_cacher(self, clear=False, verify_is_copy=True): + def _maybe_update_cacher( + self, clear: bool_t = False, verify_is_copy: bool_t = True + ) -> None: """ See if we need to update our parent cacher if clear, then clear our cache. @@ -3273,7 +3327,7 @@ def _maybe_update_cacher(self, clear=False, verify_is_copy=True): if clear: self._clear_item_cache() - def _clear_item_cache(self): + def _clear_item_cache(self) -> None: self._item_cache.clear() # ---------------------------------------------------------------------- @@ -3533,10 +3587,10 @@ class animal locomotion _xs = xs # type: Callable - def __getitem__(self, item): + def __getitem__(self, item: int): raise AbstractMethodError(self) - def _get_item_cache(self, item): + def _get_item_cache(self, item: int): """Return the cached item, item represents a label indexer.""" cache = self._item_cache res = cache.get(item) @@ -3550,7 +3604,7 @@ def _get_item_cache(self, item): res._is_copy = self._is_copy return res - def _iget_item_cache(self, item): + def _iget_item_cache(self, item: int): """Return the cached item, item represents a positional indexer.""" ax = self._info_axis if ax.is_unique: @@ -3562,7 +3616,9 @@ def _iget_item_cache(self, item): def _box_item_values(self, key, values): raise AbstractMethodError(self) - def _slice(self, slobj, axis=0, kind=None): + def _slice( + self: FrameOrSeries, slobj: slice, axis: Axis = 0, kind: Optional[str] = None + ) -> FrameOrSeries: """ Construct a slice of this container. @@ -3578,11 +3634,11 @@ def _slice(self, slobj, axis=0, kind=None): result._set_is_copy(self, copy=is_copy) return result - def _set_item(self, key, value): + def _set_item(self, key, value) -> None: self._data.set(key, value) self._clear_item_cache() - def _set_is_copy(self, ref=None, copy=True): + def _set_is_copy(self, ref=None, copy: bool_t = True) -> None: if not copy: self._is_copy = None else: @@ -3591,7 +3647,7 @@ def _set_is_copy(self, ref=None, copy=True): else: self._is_copy = None - def _check_is_chained_assignment_possible(self): + def _check_is_chained_assignment_possible(self) -> bool_t: """ Check if we are a view, have a cacher, and are of mixed type. If so, then force a setitem_copy check. @@ -3688,7 +3744,7 @@ def _check_setitem_copy(self, stacklevel=4, t="setting", force=False): elif value == "warn": warnings.warn(t, com.SettingWithCopyWarning, stacklevel=stacklevel) - def __delitem__(self, key): + def __delitem__(self, key) -> None: """ Delete item """ @@ -3745,7 +3801,7 @@ def get(self, key, default=None): return default @property - def _is_view(self): + def _is_view(self) -> bool_t: """Return boolean indicating if self is view of another array """ return self._data.is_view @@ -3892,7 +3948,13 @@ def drop( else: return obj - def _drop_axis(self, labels, axis, level=None, errors="raise"): + def _drop_axis( + self: FrameOrSeries, + labels, + axis: Axis, + level: Optional[Level] = None, + errors: Errors = "raise", + ) -> FrameOrSeries: """ Drop labels from specified axis. Used in the ``drop`` method internally. @@ -3909,32 +3971,32 @@ def _drop_axis(self, labels, axis, level=None, errors="raise"): """ axis = self._get_axis_number(axis) axis_name = self._get_axis_name(axis) - axis = self._get_axis(axis) + ax = self._get_axis(axis) - if axis.is_unique: + if ax.is_unique: if level is not None: - if not isinstance(axis, MultiIndex): + if not isinstance(ax, MultiIndex): raise AssertionError("axis must be a MultiIndex") - new_axis = axis.drop(labels, level=level, errors=errors) + new_axis = ax.drop(labels, level=level, errors=errors) else: - new_axis = axis.drop(labels, errors=errors) + new_axis = ax.drop(labels, errors=errors) result = self.reindex(**{axis_name: new_axis}) # Case for non-unique axis else: labels = ensure_object(com.index_labels_to_array(labels)) if level is not None: - if not isinstance(axis, MultiIndex): + if not isinstance(ax, MultiIndex): raise AssertionError("axis must be a MultiIndex") - indexer = ~axis.get_level_values(level).isin(labels) + indexer = ~ax.get_level_values(level).isin(labels) # GH 18561 MultiIndex.drop should raise if label is absent if errors == "raise" and indexer.all(): raise KeyError("{} not found in axis".format(labels)) else: - indexer = ~axis.isin(labels) + indexer = ~ax.isin(labels) # Check if label doesn't exist along axis - labels_missing = (axis.get_indexer_for(labels) == -1).any() + labels_missing = (ax.get_indexer_for(labels) == -1).any() if errors == "raise" and labels_missing: raise KeyError("{} not found in axis".format(labels)) @@ -3945,7 +4007,11 @@ def _drop_axis(self, labels, axis, level=None, errors="raise"): return result - def _update_inplace(self, result, verify_is_copy=True): + def _update_inplace( + self: FrameOrSeries, + result: Union[BlockManager, FrameOrSeries], + verify_is_copy: bool_t = True, + ) -> None: """ Replace self internals with result. @@ -4488,7 +4554,16 @@ def reindex(self, *args, **kwargs): axes, level, limit, tolerance, method, fill_value, copy ).__finalize__(self) - def _reindex_axes(self, axes, level, limit, tolerance, method, fill_value, copy): + def _reindex_axes( + self: FrameOrSeries, + axes: Dict[str, Any], + level: Optional[Level], + limit: Optional[int], + tolerance, + method: Optional[str], + fill_value, + copy: bool_t, + ) -> FrameOrSeries: """Perform the reindex for all the axes.""" obj = self for a in self._AXIS_ORDERS: @@ -4511,7 +4586,9 @@ def _reindex_axes(self, axes, level, limit, tolerance, method, fill_value, copy) return obj - def _needs_reindex_multi(self, axes, method, level): + def _needs_reindex_multi( + self, axes: Dict[str, Any], method: Optional[str], level: Optional[Level] + ) -> bool_t: """Check if we do need a multi reindex.""" return ( (com.count_not_none(*axes.values()) == self._AXIS_LEN) @@ -4524,8 +4601,12 @@ def _reindex_multi(self, axes, copy, fill_value): return NotImplemented def _reindex_with_indexers( - self, reindexers, fill_value=None, copy=False, allow_dups=False - ): + self: FrameOrSeries, + reindexers, + fill_value=None, + copy: bool_t = False, + allow_dups: bool_t = False, + ) -> FrameOrSeries: """allow_dups indicates an internal call here """ # reindex doing multiple operations on different axes if indicated @@ -5116,7 +5197,12 @@ def pipe(self, func, *args, **kwargs): # ---------------------------------------------------------------------- # Attribute access - def __finalize__(self, other, method=None, **kwargs): + def __finalize__( + self: FrameOrSeries, + other: FrameOrSeries, + method: Optional[str] = None, + **kwargs + ) -> FrameOrSeries: """ Propagate metadata from other to self. @@ -5152,7 +5238,7 @@ def __getattr__(self, name): return self[name] return object.__getattribute__(self, name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value) -> None: """After regular attribute access, try setting the name This allows simpler access to columns for interactive use. """ @@ -5193,7 +5279,7 @@ def __setattr__(self, name, value): ) object.__setattr__(self, name, value) - def _dir_additions(self): + def _dir_additions(self) -> Set[str]: """ add the string-like attributes from the info_axis. If info_axis is a MultiIndex, it's first level values are used. """ @@ -5207,7 +5293,7 @@ def _dir_additions(self): # ---------------------------------------------------------------------- # Consolidation of internals - def _protect_consolidate(self, f): + def _protect_consolidate(self, f: Callable[..., bool_t]) -> bool_t: """Consolidate _data -- if the blocks have changed, then clear the cache """ @@ -5217,7 +5303,7 @@ def _protect_consolidate(self, f): self._clear_item_cache() return result - def _consolidate_inplace(self): + def _consolidate_inplace(self) -> None: """Consolidate data in place and return None""" def f(): @@ -5225,7 +5311,9 @@ def f(): self._protect_consolidate(f) - def _consolidate(self, inplace=False): + def _consolidate( + self: FrameOrSeries, inplace: bool_t = False + ) -> Optional[FrameOrSeries]: """ Compute NDFrame with "consolidated" internals (data of each dtype grouped together in a single ndarray). @@ -5242,27 +5330,28 @@ def _consolidate(self, inplace=False): inplace = validate_bool_kwarg(inplace, "inplace") if inplace: self._consolidate_inplace() + return None else: f = lambda: self._data.consolidate() cons_data = self._protect_consolidate(f) return self._constructor(cons_data).__finalize__(self) @property - def _is_mixed_type(self): + def _is_mixed_type(self) -> bool_t: f = lambda: self._data.is_mixed_type return self._protect_consolidate(f) @property - def _is_numeric_mixed_type(self): + def _is_numeric_mixed_type(self) -> bool_t: f = lambda: self._data.is_numeric_mixed_type return self._protect_consolidate(f) @property - def _is_datelike_mixed_type(self): + def _is_datelike_mixed_type(self) -> bool_t: f = lambda: self._data.is_datelike_mixed_type return self._protect_consolidate(f) - def _check_inplace_setting(self, value): + def _check_inplace_setting(self, value) -> bool_t: """ check whether we allow in-place setting with this type of value """ if self._is_mixed_type: @@ -5282,10 +5371,10 @@ def _check_inplace_setting(self, value): return True - def _get_numeric_data(self): + def _get_numeric_data(self: FrameOrSeries) -> FrameOrSeries: return self._constructor(self._data.get_numeric_data()).__finalize__(self) - def _get_bool_data(self): + def _get_bool_data(self: FrameOrSeries) -> FrameOrSeries: return self._constructor(self._data.get_bool_data()).__finalize__(self) # ---------------------------------------------------------------------- @@ -5701,7 +5790,9 @@ def blocks(self): """ return self.as_blocks() - def _to_dict_of_blocks(self, copy=True): + def _to_dict_of_blocks( + self: FrameOrSeries, copy: bool_t = True + ) -> Dict[str, FrameOrSeries]: """ Return a dict of dtype -> Constructor Types that each is a homogeneous dtype. @@ -5970,10 +6061,10 @@ def copy(self, deep=True): data = self._data.copy(deep=deep) return self._constructor(data).__finalize__(self) - def __copy__(self, deep=True): + def __copy__(self: FrameOrSeries, deep: bool_t = True) -> FrameOrSeries: return self.copy(deep=deep) - def __deepcopy__(self, memo=None): + def __deepcopy__(self: FrameOrSeries, memo=None) -> FrameOrSeries: """ Parameters ---------- @@ -5985,8 +6076,13 @@ def __deepcopy__(self, memo=None): return self.copy(deep=True) def _convert( - self, datetime=False, numeric=False, timedelta=False, coerce=False, copy=True - ): + self: FrameOrSeries, + datetime: bool_t = False, + numeric: bool_t = False, + timedelta: bool_t = False, + coerce: bool_t = False, + copy: bool_t = True, + ) -> FrameOrSeries: """ Attempt to infer better dtype for object columns @@ -7361,7 +7457,12 @@ def notna(self): def notnull(self): return notna(self).__finalize__(self) - def _clip_with_scalar(self, lower, upper, inplace=False): + def _clip_with_scalar( + self: FrameOrSeries, + lower: Optional[float], + upper: Optional[float], + inplace: bool_t = False, + ) -> Optional[FrameOrSeries]: if (lower is not None and np.any(isna(lower))) or ( upper is not None and np.any(isna(upper)) ): @@ -7383,10 +7484,17 @@ def _clip_with_scalar(self, lower, upper, inplace=False): if inplace: self._update_inplace(result) + return None else: return result - def _clip_with_one_bound(self, threshold, method, axis, inplace): + def _clip_with_one_bound( + self: FrameOrSeries, + threshold, + method: Callable, + axis: Optional[Axis], + inplace: bool_t, + ) -> Optional[FrameOrSeries]: if axis is not None: axis = self._get_axis_number(axis) @@ -8793,16 +8901,16 @@ def align( def _align_frame( self, - other, - join="outer", - axis=None, - level=None, - copy=True, + other: "DataFrame", + join: str = "outer", + axis: Optional[int] = None, + level: Optional[Level] = None, + copy: bool_t = True, fill_value=None, - method=None, - limit=None, - fill_axis=0, - ): + method: Optional[str] = None, + limit: Optional[int] = None, + fill_axis: Axis = 0, + ) -> Tuple["NDFrame", "NDFrame"]: # defaults join_index, join_columns = None, None ilidx, iridx = None, None @@ -8853,16 +8961,16 @@ def _align_frame( def _align_series( self, - other, - join="outer", - axis=None, - level=None, - copy=True, + other: "Series", + join: str = "outer", + axis: Optional[int] = None, + level: Optional[Level] = None, + copy: bool_t = True, fill_value=None, method=None, limit=None, - fill_axis=0, - ): + fill_axis: Axis = 0, + ) -> Tuple["NDFrame", "NDFrame"]: is_series = isinstance(self, ABCSeries) @@ -8936,15 +9044,15 @@ def _align_series( return left.__finalize__(self), right.__finalize__(other) def _where( - self, + self: FrameOrSeries, cond, other=np.nan, - inplace=False, - axis=None, - level=None, - errors="raise", - try_cast=False, - ): + inplace: bool_t = False, + axis: Optional[Axis] = None, + level: Optional[Level] = None, + errors: Errors = "raise", + try_cast: bool_t = False, + ) -> Optional[FrameOrSeries]: """ Equivalent to public method `where`, except that `other` is not applied as a function even if callable. Used in __setitem__. @@ -9078,6 +9186,7 @@ def _where( transpose=self._AXIS_REVERSED, ) self._update_inplace(new_data) + return None else: new_data = self._data.where( @@ -10392,7 +10501,14 @@ def pct_change(self, periods=1, fill_method="pad", limit=None, freq=None, **kwar np.putmask(rs.values, mask, np.nan) return rs - def _agg_by_level(self, name, axis=0, level=0, skipna=True, **kwargs): + def _agg_by_level( + self: FrameOrSeries, + name: str, + axis: Axis = 0, + level: Level = 0, + skipna: bool_t = True, + **kwargs + ) -> FrameOrSeries: if axis is None: raise ValueError("Must specify 'axis' when aggregating by level.") grouped = self.groupby(level=level, axis=axis, sort=False) @@ -10791,7 +10907,7 @@ def transform(self, func, *args, **kwargs): Also returns None for empty %(klass)s. """ - def _find_valid_index(self, how): + def _find_valid_index(self, how: How): """ Retrieves the index of the first valid value. diff --git a/pandas/core/series.py b/pandas/core/series.py index f840b6ce649b8..23c5764439bda 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -54,6 +54,7 @@ ) import pandas as pd +from pandas._typing import Axis from pandas.core import algorithms, base, generic, nanops, ops from pandas.core.accessor import CachedAccessor from pandas.core.arrays import ExtensionArray, SparseArray @@ -1077,7 +1078,7 @@ def _ixs(self, i: int, axis: int = 0): else: return values[i] - def _slice(self, slobj: slice, axis: int = 0, kind=None): + def _slice(self, slobj: slice, axis: Axis = 0, kind=None): slobj = self.index._convert_slice_indexer(slobj, kind=kind or "getitem") return self._get_values(slobj) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 6e4894bdb0f56..788ca8369682f 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -643,7 +643,7 @@ def _to_str_columns(self) -> List[List[str]]: if not is_list_like(self.header) and not self.header: stringified = [] - for i, c in enumerate(frame): + for i, _ in enumerate(frame): fmt_values = self._format_col(i) fmt_values = _make_fixed_width( fmt_values, @@ -674,7 +674,7 @@ def _to_str_columns(self) -> List[List[str]]: x.append("") stringified = [] - for i, c in enumerate(frame): + for i, _ in enumerate(frame): cheader = str_columns[i] header_colwidth = max( self.col_space or 0, *(self.adj.len(x) for x in cheader) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index e375bd459e66f..6677d08dbcc51 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1170,16 +1170,8 @@ def test_extension_array_cross_section_converts(): "idxr, error, error_message", [ (lambda x: x, AbstractMethodError, None), - ( - lambda x: x.loc, - AttributeError, - "type object 'NDFrame' has no attribute '_AXIS_ALIASES'", - ), - ( - lambda x: x.iloc, - AttributeError, - "type object 'NDFrame' has no attribute '_AXIS_ALIASES'", - ), + (lambda x: x.loc, AttributeError, "'NoneType' object has no attribute 'get'"), + (lambda x: x.iloc, AttributeError, "'NoneType' object has no attribute 'get'"), ], ) def test_ndframe_indexing_raises(idxr, error, error_message): diff --git a/requirements-dev.txt b/requirements-dev.txt index e49ad10bfc99d..467f914ceddbb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,6 +11,7 @@ flake8-rst>=0.6.0,<=0.7.0 isort mypy pycodestyle +typing_extensions gitpython sphinx==1.8.5 numpydoc>=0.9.0 From a20701036cb5cbd024d7e2d731dcd6597d374778 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 29 Jul 2019 20:35:16 +0100 Subject: [PATCH 02/11] black --- pandas/core/generic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index b12d7202d0768..ac7a25b7e453e 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -89,6 +89,7 @@ if TYPE_CHECKING: from pandas import Series, DataFrame from typing_extensions import Literal + Errors = Literal["ignore", "raise"] How = Literal["first", "last"] else: From 3ba8c764b2a0b2d6663b11812f897a83aa667ba4 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 31 Jul 2019 14:48:08 +0100 Subject: [PATCH 03/11] move typing_extensions to _typing.py --- pandas/_typing.py | 7 +++++++ pandas/core/generic.py | 15 ++++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index df8367cc1ab89..55f310be5f2d0 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -14,6 +14,13 @@ from pandas.core.frame import DataFrame # noqa: F401 from pandas.core.series import Series # noqa: F401 from pandas.core.sparse.series import SparseSeries # noqa: F401 + from typing_extensions import Literal + + IgnoreRaise = Literal["ignore", "raise"] + FirstLast = Literal["first", "last"] +else: + IgnoreRaise = str + FirstLast = str AnyArrayLike = TypeVar( diff --git a/pandas/core/generic.py b/pandas/core/generic.py index ac7a25b7e453e..7932f5bb119d6 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -64,7 +64,7 @@ from pandas.core.dtypes.missing import isna, notna import pandas as pd -from pandas._typing import Axis, Dtype, Level +from pandas._typing import Axis, Dtype, FirstLast, IgnoreRaise, Level from pandas.core import missing, nanops import pandas.core.algorithms as algos from pandas.core.base import PandasObject, SelectionMixin @@ -88,13 +88,6 @@ if TYPE_CHECKING: from pandas import Series, DataFrame - from typing_extensions import Literal - - Errors = Literal["ignore", "raise"] - How = Literal["first", "last"] -else: - Errors = str - How = str bool_t = bool FrameOrSeries = TypeVar("FrameOrSeries", bound="NDFrame") @@ -3953,7 +3946,7 @@ def _drop_axis( labels, axis: Axis, level: Optional[Level] = None, - errors: Errors = "raise", + errors: IgnoreRaise = "raise", ) -> FrameOrSeries: """ Drop labels from specified axis. Used in the ``drop`` method @@ -9042,7 +9035,7 @@ def _where( inplace: bool_t = False, axis: Optional[Axis] = None, level: Optional[Level] = None, - errors: Errors = "raise", + errors: IgnoreRaise = "raise", try_cast: bool_t = False, ) -> Optional[FrameOrSeries]: """ @@ -10899,7 +10892,7 @@ def transform(self, func, *args, **kwargs): Also returns None for empty %(klass)s. """ - def _find_valid_index(self, how: How): + def _find_valid_index(self, how: FirstLast): """ Retrieves the index of the first valid value. From 9aa69fe5491e0b161e679c8f54cbb8f13a48b85a Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 31 Jul 2019 15:03:54 +0100 Subject: [PATCH 04/11] add whatsnew --- doc/source/whatsnew/v1.0.0.rst | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index e2f7bb47ca34b..0877ea1bc2d34 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -42,6 +42,45 @@ Backwards incompatible API changes - :class:`pandas.core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`). - +.. _whatsnew_1000.api_breaking.deps: + +Increased minimum versions for dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Supporting Python 3.6, we are not able to take advantage of new types added to the typing module. The typing_extensions module +contains both backports of these changes as well as experimental types that will eventually be added to the typing module. + +If installed, we now require: + ++-------------------------+-----------------+----------+ +| Package | Minimum Version | Required | ++=========================+=================+==========+ +| | | | ++-------------------------+-----------------+----------+ +| | | | ++-------------------------+-----------------+----------+ +| typing_extensions (dev) | | | ++-------------------------+-----------------+----------+ + +For `optional libraries `_ the general recommendation is to use the latest version. +The following table lists the lowest version per library that is currently being tested throughout the development of pandas. +Optional libraries below the lowest tested version may still work, but are not considered supported. + ++-----------------+-----------------+ +| Package | Minimum Version | ++=================+=================+ +| | | ++-----------------+-----------------+ +| | | ++-----------------+-----------------+ +| | | ++-----------------+-----------------+ +| | | ++-----------------+-----------------+ + +See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more. + + Other API changes ^^^^^^^^^^^^^^^^^ From 8fb0254b456b5e9a608de424ea78cbfb42c7a0bf Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 3 Aug 2019 10:14:42 +0100 Subject: [PATCH 05/11] use Literal for Axis Alias --- pandas/_typing.py | 8 ++++++-- pandas/core/generic.py | 30 ++++++++++++++++++------------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index 55f310be5f2d0..5fcd025fbea27 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -18,10 +18,15 @@ IgnoreRaise = Literal["ignore", "raise"] FirstLast = Literal["first", "last"] + AxisInt = Literal[0, 1] + AxisStr = Literal["index", "columns"] + Axis = Literal[AxisInt, AxisStr] else: IgnoreRaise = str FirstLast = str - + AxisInt = int + AxisStr = str + Axis = Union[str, int] AnyArrayLike = TypeVar( "AnyArrayLike", "ExtensionArray", "Index", "Series", "SparseSeries", np.ndarray @@ -33,6 +38,5 @@ FrameOrSeries = TypeVar("FrameOrSeries", "Series", "DataFrame") Scalar = Union[str, int, float] -Axis = Union[str, int] Ordered = Optional[bool] Level = Union[str, int] diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 6f72daeeb0aee..72e401a61740a 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -21,6 +21,7 @@ Type, TypeVar, Union, + cast, ) import warnings import weakref @@ -64,7 +65,7 @@ from pandas.core.dtypes.missing import isna, notna import pandas as pd -from pandas._typing import Axis, Dtype, FirstLast, IgnoreRaise, Level +from pandas._typing import Axis, AxisInt, AxisStr, Dtype, FirstLast, IgnoreRaise, Level from pandas.core import missing, nanops import pandas.core.algorithms as algos from pandas.core.base import PandasObject, SelectionMixin @@ -176,9 +177,9 @@ class NDFrame(PandasObject, SelectionMixin): _metadata = [] # type: List[str] _is_copy = None _data = None # type: BlockManager - _AXIS_ALIASES = None # type: Dict[str, int] - _AXIS_NAMES = None # type: Dict[int, str] - _AXIS_NUMBERS = None # type: Dict[str, int] + _AXIS_ALIASES = None # type: Dict[AxisStr, AxisInt] + _AXIS_NAMES = None # type: Dict[AxisInt, AxisStr] + _AXIS_NUMBERS = None # type: Dict[AxisStr, AxisInt] _AXIS_REVERSED = None _AXIS_LEN = None # type: int @@ -209,7 +210,11 @@ def __init__( object.__setattr__(self, "_item_cache", {}) def _init_mgr( - self, mgr: BlockManager, axes: Dict[str, Any], dtype=None, copy: bool = False + self, + mgr: BlockManager, + axes: Dict[AxisStr, Any], + dtype=None, + copy: bool = False, ) -> BlockManager: """ passed a manager and a axes dict """ for a, axe in axes.items(): @@ -359,7 +364,7 @@ def set_axis(a, i): assert not isinstance(ns, dict) def _construct_axes_dict( - self, axes: Optional[Iterable[str]] = None, **kwargs + self, axes: Optional[Iterable[AxisStr]] = None, **kwargs ) -> Dict[str, Index]: """Return an axes dictionary for myself.""" d = {a: self._get_axis(a) for a in (axes or self._AXIS_ORDERS)} @@ -434,7 +439,7 @@ def _from_axes( return cls(data, **d) @classmethod - def _get_axis_number(cls, axis) -> int: + def _get_axis_number(cls, axis) -> AxisInt: axis = cls._AXIS_ALIASES.get(axis, axis) if is_integer(axis): if axis in cls._AXIS_NAMES: @@ -447,9 +452,10 @@ def _get_axis_number(cls, axis) -> int: raise ValueError("No axis named {0} for object type {1}".format(axis, cls)) @classmethod - def _get_axis_name(cls, axis) -> str: + def _get_axis_name(cls, axis) -> AxisStr: axis = cls._AXIS_ALIASES.get(axis, axis) if isinstance(axis, str): + axis = cast(AxisStr, axis) if axis in cls._AXIS_NUMBERS: return axis else: @@ -464,7 +470,7 @@ def _get_axis(self, axis: Axis) -> Index: return getattr(self, name) @classmethod - def _get_block_manager_axis(cls, axis: Axis) -> int: + def _get_block_manager_axis(cls, axis: Axis) -> AxisInt: """Map the axis to the block_manager axis.""" axis = cls._get_axis_number(axis) if cls._AXIS_REVERSED: @@ -701,7 +707,7 @@ def set_axis(self, labels, axis=0, inplace=False): obj.set_axis(labels, axis=axis, inplace=True) return obj - def _set_axis(self, axis: int, labels) -> None: + def _set_axis(self, axis: AxisInt, labels) -> None: self._data.set_axis(axis, labels) self._clear_item_cache() @@ -1779,7 +1785,7 @@ def _get_label_or_level_values(self, key, axis: Axis = 0): future version """ axis = self._get_axis_number(axis) - other_axes = [ax for ax in range(self._AXIS_LEN) if ax != axis] + other_axes = [cast(AxisInt, ax) for ax in range(self._AXIS_LEN) if ax != axis] if self._is_label_reference(key, axis=axis): self._check_label_or_level_ambiguity(key, axis=axis) @@ -3970,7 +3976,7 @@ def _drop_axis( new_axis = ax.drop(labels, level=level, errors=errors) else: new_axis = ax.drop(labels, errors=errors) - result = self.reindex(**{axis_name: new_axis}) + result = self.reindex(**{cast(str, axis_name): new_axis}) # Case for non-unique axis else: From 3b042211291b65ce9dbb1e75d2d29df4df5e3bb2 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 3 Aug 2019 18:36:24 +0100 Subject: [PATCH 06/11] return type Type[FrameOrSeries] for _constructor --- pandas/core/generic.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 72e401a61740a..71b654eed9eca 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -92,6 +92,7 @@ bool_t = bool FrameOrSeries = TypeVar("FrameOrSeries", bound="NDFrame") +_T = TypeVar("_T") # goal is to be able to define the docs close to function, while still being # able to share @@ -276,7 +277,7 @@ def _validate_dtype(self, dtype): # Construction @property - def _constructor(self: FrameOrSeries) -> FrameOrSeries: + def _constructor(self: FrameOrSeries) -> Type[FrameOrSeries]: """Used when a manipulation result has the same dimensions as the original. """ @@ -2020,7 +2021,8 @@ def __array__(self, dtype=None): def __array_wrap__(self: FrameOrSeries, result, context=None) -> FrameOrSeries: d = self._construct_axes_dict(self._AXIS_ORDERS, copy=False) - return self._constructor(result, **d).__finalize__(self) + # https://github.com/python/mypy/issues/5382 + return self._constructor(result, **d).__finalize__(self) # type: ignore # ideally we would define this to avoid the getattr checks, but # is slower @@ -5287,7 +5289,7 @@ def _dir_additions(self) -> Set[str]: # ---------------------------------------------------------------------- # Consolidation of internals - def _protect_consolidate(self, f: Callable[..., bool_t]) -> bool_t: + def _protect_consolidate(self, f: Callable[..., _T]) -> _T: """Consolidate _data -- if the blocks have changed, then clear the cache """ @@ -9056,7 +9058,10 @@ def _where( cond = np.asanyarray(cond) if cond.shape != self.shape: raise ValueError("Array conditional must be same shape as self") - cond = self._constructor(cond, **self._construct_axes_dict()) + # https://github.com/python/mypy/issues/5382 + cond = self._constructor( # type: ignore + cond, **self._construct_axes_dict() + ) # make sure we are boolean fill_value = bool(inplace) @@ -9148,7 +9153,10 @@ def _where( # we are the same shape, so create an actual object for alignment else: - other = self._constructor(other, **self._construct_axes_dict()) + # https://github.com/python/mypy/issues/5382 + other = self._constructor( # type: ignore + other, **self._construct_axes_dict() + ) if axis is None: axis = 0 From 7c4e87a74395db30e4ab388a477e64b85164aeee Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 6 Aug 2019 21:17:37 +0100 Subject: [PATCH 07/11] remove typing_extensions --- doc/source/whatsnew/v1.0.0.rst | 39 ---------------------------------- environment.yml | 1 - pandas/_typing.py | 13 +----------- pandas/core/generic.py | 36 +++++++++++++------------------ requirements-dev.txt | 1 - 5 files changed, 16 insertions(+), 74 deletions(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index e0e2bec5aa44b..c7f8bb70e3461 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -42,45 +42,6 @@ Backwards incompatible API changes - :class:`pandas.core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`). - -.. _whatsnew_1000.api_breaking.deps: - -Increased minimum versions for dependencies -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Supporting Python 3.6, we are not able to take advantage of new types added to the typing module. The typing_extensions module -contains both backports of these changes as well as experimental types that will eventually be added to the typing module. - -If installed, we now require: - -+-------------------------+-----------------+----------+ -| Package | Minimum Version | Required | -+=========================+=================+==========+ -| | | | -+-------------------------+-----------------+----------+ -| | | | -+-------------------------+-----------------+----------+ -| typing_extensions (dev) | | | -+-------------------------+-----------------+----------+ - -For `optional libraries `_ the general recommendation is to use the latest version. -The following table lists the lowest version per library that is currently being tested throughout the development of pandas. -Optional libraries below the lowest tested version may still work, but are not considered supported. - -+-----------------+-----------------+ -| Package | Minimum Version | -+=================+=================+ -| | | -+-----------------+-----------------+ -| | | -+-----------------+-----------------+ -| | | -+-----------------+-----------------+ -| | | -+-----------------+-----------------+ - -See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more. - - Other API changes ^^^^^^^^^^^^^^^^^ diff --git a/environment.yml b/environment.yml index 4deeeff81df59..93e8302b498a0 100644 --- a/environment.yml +++ b/environment.yml @@ -24,7 +24,6 @@ dependencies: - isort # check that imports are in the right order - mypy - pycodestyle # used by flake8 - - typing_extensions # documentation - gitpython # obtain contributors from git for whatsnew diff --git a/pandas/_typing.py b/pandas/_typing.py index 5fcd025fbea27..df8367cc1ab89 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -14,19 +14,7 @@ from pandas.core.frame import DataFrame # noqa: F401 from pandas.core.series import Series # noqa: F401 from pandas.core.sparse.series import SparseSeries # noqa: F401 - from typing_extensions import Literal - IgnoreRaise = Literal["ignore", "raise"] - FirstLast = Literal["first", "last"] - AxisInt = Literal[0, 1] - AxisStr = Literal["index", "columns"] - Axis = Literal[AxisInt, AxisStr] -else: - IgnoreRaise = str - FirstLast = str - AxisInt = int - AxisStr = str - Axis = Union[str, int] AnyArrayLike = TypeVar( "AnyArrayLike", "ExtensionArray", "Index", "Series", "SparseSeries", np.ndarray @@ -38,5 +26,6 @@ FrameOrSeries = TypeVar("FrameOrSeries", "Series", "DataFrame") Scalar = Union[str, int, float] +Axis = Union[str, int] Ordered = Optional[bool] Level = Union[str, int] diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 856d2adb9af1a..ecf3f84848ab5 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -21,7 +21,6 @@ Type, TypeVar, Union, - cast, ) import warnings import weakref @@ -65,7 +64,7 @@ from pandas.core.dtypes.missing import isna, notna import pandas as pd -from pandas._typing import Axis, AxisInt, AxisStr, Dtype, FirstLast, IgnoreRaise, Level +from pandas._typing import Axis, Dtype, Level from pandas.core import missing, nanops import pandas.core.algorithms as algos from pandas.core.base import PandasObject, SelectionMixin @@ -179,9 +178,9 @@ class NDFrame(PandasObject, SelectionMixin): _metadata = [] # type: List[str] _is_copy = None _data = None # type: BlockManager - _AXIS_ALIASES = None # type: Dict[AxisStr, AxisInt] - _AXIS_NAMES = None # type: Dict[AxisInt, AxisStr] - _AXIS_NUMBERS = None # type: Dict[AxisStr, AxisInt] + _AXIS_ALIASES = None # type: Dict[str, int] + _AXIS_NAMES = None # type: Dict[int, str] + _AXIS_NUMBERS = None # type: Dict[str, int] _AXIS_REVERSED = None _AXIS_LEN = None # type: int @@ -212,11 +211,7 @@ def __init__( object.__setattr__(self, "_item_cache", {}) def _init_mgr( - self, - mgr: BlockManager, - axes: Dict[AxisStr, Any], - dtype=None, - copy: bool = False, + self, mgr: BlockManager, axes: Dict[str, Any], dtype=None, copy: bool = False ) -> BlockManager: """ passed a manager and a axes dict """ for a, axe in axes.items(): @@ -366,7 +361,7 @@ def set_axis(a, i): assert not isinstance(ns, dict) def _construct_axes_dict( - self, axes: Optional[Iterable[AxisStr]] = None, **kwargs + self, axes: Optional[Iterable[str]] = None, **kwargs ) -> Dict[str, Index]: """Return an axes dictionary for myself.""" d = {a: self._get_axis(a) for a in (axes or self._AXIS_ORDERS)} @@ -441,7 +436,7 @@ def _from_axes( return cls(data, **d) @classmethod - def _get_axis_number(cls, axis) -> AxisInt: + def _get_axis_number(cls, axis) -> int: axis = cls._AXIS_ALIASES.get(axis, axis) if is_integer(axis): if axis in cls._AXIS_NAMES: @@ -454,10 +449,9 @@ def _get_axis_number(cls, axis) -> AxisInt: raise ValueError("No axis named {0} for object type {1}".format(axis, cls)) @classmethod - def _get_axis_name(cls, axis) -> AxisStr: + def _get_axis_name(cls, axis) -> str: axis = cls._AXIS_ALIASES.get(axis, axis) if isinstance(axis, str): - axis = cast(AxisStr, axis) if axis in cls._AXIS_NUMBERS: return axis else: @@ -472,7 +466,7 @@ def _get_axis(self, axis: Axis) -> Index: return getattr(self, name) @classmethod - def _get_block_manager_axis(cls, axis: Axis) -> AxisInt: + def _get_block_manager_axis(cls, axis: Axis) -> int: """Map the axis to the block_manager axis.""" axis = cls._get_axis_number(axis) if cls._AXIS_REVERSED: @@ -709,7 +703,7 @@ def set_axis(self, labels, axis=0, inplace=False): obj.set_axis(labels, axis=axis, inplace=True) return obj - def _set_axis(self, axis: AxisInt, labels) -> None: + def _set_axis(self, axis: int, labels) -> None: self._data.set_axis(axis, labels) self._clear_item_cache() @@ -1787,7 +1781,7 @@ def _get_label_or_level_values(self, key, axis: Axis = 0): future version """ axis = self._get_axis_number(axis) - other_axes = [cast(AxisInt, ax) for ax in range(self._AXIS_LEN) if ax != axis] + other_axes = [ax for ax in range(self._AXIS_LEN) if ax != axis] if self._is_label_reference(key, axis=axis): self._check_label_or_level_ambiguity(key, axis=axis) @@ -3947,7 +3941,7 @@ def _drop_axis( labels, axis: Axis, level: Optional[Level] = None, - errors: IgnoreRaise = "raise", + errors: str = "raise", ) -> FrameOrSeries: """ Drop labels from specified axis. Used in the ``drop`` method @@ -3974,7 +3968,7 @@ def _drop_axis( new_axis = ax.drop(labels, level=level, errors=errors) else: new_axis = ax.drop(labels, errors=errors) - result = self.reindex(**{cast(str, axis_name): new_axis}) + result = self.reindex(**{axis_name: new_axis}) # Case for non-unique axis else: @@ -9033,7 +9027,7 @@ def _where( inplace: bool_t = False, axis: Optional[Axis] = None, level: Optional[Level] = None, - errors: IgnoreRaise = "raise", + errors: str = "raise", try_cast: bool_t = False, ) -> Optional[FrameOrSeries]: """ @@ -10896,7 +10890,7 @@ def transform(self, func, *args, **kwargs): Also returns None for empty %(klass)s. """ - def _find_valid_index(self, how: FirstLast): + def _find_valid_index(self, how: str): """ Retrieves the index of the first valid value. diff --git a/requirements-dev.txt b/requirements-dev.txt index 467f914ceddbb..e49ad10bfc99d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,6 @@ flake8-rst>=0.6.0,<=0.7.0 isort mypy pycodestyle -typing_extensions gitpython sphinx==1.8.5 numpydoc>=0.9.0 From 3690f0b28ac9898c24e7ae4f0bbfccadaf1ec34d Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 27 Aug 2019 20:58:04 +0100 Subject: [PATCH 08/11] remove extra line --- pandas/core/generic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 2ff4df9cea6c1..47fbabb102a33 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -89,7 +89,6 @@ from pandas.io.formats.printing import pprint_thing from pandas.tseries.frequencies import to_offset - if TYPE_CHECKING: from pandas import Series, DataFrame From aa713270aedc741077a08deaac777f9da5a31ed8 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 28 Aug 2019 12:47:09 +0100 Subject: [PATCH 09/11] import FrameOrSeries and _T from _typing --- pandas/core/generic.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 32644287c649d..b8932f8248bbe 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -66,7 +66,7 @@ from pandas.core.dtypes.missing import isna, notna import pandas as pd -from pandas._typing import Axis, Dtype, FilePathOrBuffer, Level +from pandas._typing import _T, Axis, Dtype, FilePathOrBuffer, FrameOrSeries, Level from pandas.core import missing, nanops import pandas.core.algorithms as algos from pandas.core.base import PandasObject, SelectionMixin @@ -92,9 +92,6 @@ if TYPE_CHECKING: from pandas import Series, DataFrame -FrameOrSeries = TypeVar("FrameOrSeries", bound="NDFrame") -_T = TypeVar("_T") - # goal is to be able to define the docs close to function, while still being # able to share _shared_docs = dict() # type: Dict[str, str] From 6397422a3859b68412ebd3ab1be14e17195ee1d7 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 28 Aug 2019 13:13:54 +0100 Subject: [PATCH 10/11] lint error - unused import --- pandas/core/generic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index b8932f8248bbe..b377592c175d4 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -21,7 +21,6 @@ Set, Tuple, Type, - TypeVar, Union, ) import warnings From b8866d633967f2f650c4c3e2cd2dd7a58fc56a1a Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Thu, 29 Aug 2019 14:51:56 +0100 Subject: [PATCH 11/11] revert changes to runtime behaviour --- pandas/core/generic.py | 11 ++++++----- pandas/tests/indexing/test_indexing.py | 12 ++++++++++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index b377592c175d4..05ea2c5d42be4 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -178,11 +178,12 @@ class NDFrame(PandasObject, SelectionMixin): _metadata = [] # type: List[str] _is_copy = None _data = None # type: BlockManager - _AXIS_ALIASES = None # type: Dict[str, int] - _AXIS_NAMES = None # type: Dict[int, str] - _AXIS_NUMBERS = None # type: Dict[str, int] - _AXIS_REVERSED = None - _AXIS_LEN = None # type: int + if TYPE_CHECKING: + _AXIS_ALIASES = None # type: Dict[str, int] + _AXIS_NAMES = None # type: Dict[int, str] + _AXIS_NUMBERS = None # type: Dict[str, int] + _AXIS_REVERSED = None + _AXIS_LEN = None # type: int # ---------------------------------------------------------------------- # Constructors diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 6677d08dbcc51..e375bd459e66f 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1170,8 +1170,16 @@ def test_extension_array_cross_section_converts(): "idxr, error, error_message", [ (lambda x: x, AbstractMethodError, None), - (lambda x: x.loc, AttributeError, "'NoneType' object has no attribute 'get'"), - (lambda x: x.iloc, AttributeError, "'NoneType' object has no attribute 'get'"), + ( + lambda x: x.loc, + AttributeError, + "type object 'NDFrame' has no attribute '_AXIS_ALIASES'", + ), + ( + lambda x: x.iloc, + AttributeError, + "type object 'NDFrame' has no attribute '_AXIS_ALIASES'", + ), ], ) def test_ndframe_indexing_raises(idxr, error, error_message):