From 1ce302064e3559589998d80113bbe6a34ee4e9ae Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:01:06 +0100 Subject: [PATCH 01/35] autogenerate type hints using monkeytype --- .gitignore | 3 + pandas/io/formats/format.py | 212 +++++++++++++++++++----------------- 2 files changed, 117 insertions(+), 98 deletions(-) diff --git a/.gitignore b/.gitignore index 56828fa1d9331..e85da9c9b976b 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,9 @@ coverage_html_report # hypothesis test database .hypothesis/ __pycache__ +# pytest-monkeytype +monkeytype.sqlite3 + # OS generated files # ###################### diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 11b69064723c5..7f77400cd2dcf 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -48,6 +48,22 @@ from pandas.io.common import _expand_user, _stringify_path from pandas.io.formats.printing import adjoin, justify, pprint_thing +from io import StringIO, TextIOWrapper +from _pytest.capture import EncodedFile +from dateutil.tz.tz import tzutc +from dateutil.zoneinfo import tzfile +from numpy import float64, int32, ndarray, str_ +from pandas._libs.tslibs.nattype import NaTType +from pandas._libs.tslibs.timestamps import Timestamp +from pandas.core.arrays.datetimes import DatetimeArray +from pandas.core.arrays.timedeltas import TimedeltaArray +from pandas.core.frame import DataFrame +from pandas.core.indexes.base import Index +from pandas.core.indexes.timedeltas import TimedeltaIndex +from pandas.core.series import Series +from pandas.tests.io.test_common import CustomFSPath +from py._path.local import LocalPath +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union if TYPE_CHECKING: from pandas import Series, DataFrame, Categorical @@ -137,8 +153,8 @@ def __init__( buf: Optional[TextIO] = None, length: bool = True, na_rep: str = "NaN", - footer: bool = True, - ): + footer: bool = True + ) -> None: self.categorical = categorical self.buf = buf if buf is not None else StringIO("") self.na_rep = na_rep @@ -206,8 +222,8 @@ def __init__( float_format: Optional[str] = None, dtype: bool = True, max_rows: Optional[int] = None, - min_rows: Optional[int] = None, - ): + min_rows: Optional[int] = None + ) -> None: self.series = series self.buf = buf if buf is not None else StringIO() self.name = name @@ -247,7 +263,7 @@ def _chk_truncate(self) -> None: else: row_num = max_rows // 2 series = concat((series.iloc[:row_num], series.iloc[-row_num:])) - self.tr_row_num = row_num # type: Optional[int] + self.tr_row_num: Optional[int] = row_num else: self.tr_row_num = None self.tr_series = series @@ -351,21 +367,21 @@ def to_string(self) -> str: class TextAdjustment: - def __init__(self): + def __init__(self) -> None: self.encoding = get_option("display.encoding") - def len(self, text): + def len(self, text: Union[str_, str]) -> int: return len(text) - def justify(self, texts, max_len, mode="right"): + def justify(self, texts: Any, max_len: int, mode: str = "right") -> List[str]: return justify(texts, max_len, mode=mode) - def adjoin(self, space, *lists, **kwargs): + def adjoin(self, space: int, *lists, **kwargs) -> str: return adjoin(space, *lists, strlen=self.len, justfunc=self.justify, **kwargs) class EastAsianTextAdjustment(TextAdjustment): - def __init__(self): + def __init__(self) -> None: super().__init__() if get_option("display.unicode.ambiguous_as_wide"): self.ambiguous_width = 2 @@ -377,7 +393,7 @@ def __init__(self): # Ambiguous width can be changed by option self._EAW_MAP = {"Na": 1, "N": 1, "W": 2, "F": 2, "H": 1} - def len(self, text): + def len(self, text: str) -> int: """ Calculate display width considering unicode East Asian Width """ @@ -388,7 +404,7 @@ def len(self, text): self._EAW_MAP.get(east_asian_width(c), self.ambiguous_width) for c in text ) - def justify(self, texts, max_len, mode="right"): + def justify(self, texts: Union[Tuple[str, str], List[str], Tuple[str, str, str, str, str], Tuple[str, str, str, str]], max_len: int, mode: str = "right") -> List[str]: # re-calculate padding space per str considering East Asian Width def _get_pad(t): return max_len - self.len(t) + len(t) @@ -401,7 +417,7 @@ def _get_pad(t): return [x.rjust(_get_pad(x)) for x in texts] -def _get_adjustment(): +def _get_adjustment() -> Union[TextAdjustment, EastAsianTextAdjustment]: use_east_asian_width = get_option("display.unicode.east_asian_width") if use_east_asian_width: return EastAsianTextAdjustment() @@ -414,12 +430,12 @@ class TableFormatter: show_dimensions = None @property - def should_show_dimensions(self): + def should_show_dimensions(self) -> Optional[bool]: return self.show_dimensions is True or ( self.show_dimensions == "truncate" and self.is_truncated ) - def _get_formatter(self, i): + def _get_formatter(self, i: Union[str, int]) -> Optional[Callable]: if isinstance(self.formatters, (list, tuple)): if is_integer(i): return self.formatters[i] @@ -446,28 +462,28 @@ class DataFrameFormatter(TableFormatter): def __init__( self, - frame, - buf=None, - columns=None, - col_space=None, - header=True, - index=True, - na_rep="NaN", - formatters=None, - justify=None, - float_format=None, - sparsify=None, - index_names=True, - line_width=None, - max_rows=None, - min_rows=None, - max_cols=None, - show_dimensions=False, - decimal=".", - table_id=None, - render_links=False, + frame: DataFrame, + buf: Optional[Union[StringIO, LocalPath, str, CustomFSPath]] = None, + columns: Optional[List[str]] = None, + col_space: Optional[Union[str, int]] = None, + header: Union[bool, List[str]] = True, + index: Union[bool, int] = True, + na_rep: str = "NaN", + formatters: Optional[Union[Tuple[Callable, Callable, Callable], Dict[str, Callable]]] = None, + justify: Optional[str] = None, + float_format: Optional[Union[str, Type[str]]] = None, + sparsify: Optional[bool] = None, + index_names: bool = True, + line_width: Optional[int] = None, + max_rows: Optional[int] = None, + min_rows: Optional[int] = None, + max_cols: Optional[int] = None, + show_dimensions: Union[bool, str] = False, + decimal: str = ".", + table_id: Optional[str] = None, + render_links: bool = False, **kwds - ): + ) -> None: self.frame = frame if buf is not None: self.buf = _expand_user(_stringify_path(buf)) @@ -742,7 +758,7 @@ def to_string(self) -> None: ) ) - def _join_multiline(self, *strcols): + def _join_multiline(self, *strcols) -> str: lwidth = self.line_width adjoin_width = 1 strcols = list(strcols) @@ -779,13 +795,13 @@ def _join_multiline(self, *strcols): def to_latex( self, - column_format=None, - longtable=False, - encoding=None, - multicolumn=False, - multicolumn_format=None, - multirow=False, - ): + column_format: Optional[str] = None, + longtable: bool = False, + encoding: Optional[str] = None, + multicolumn: bool = False, + multicolumn_format: Optional[str] = None, + multirow: bool = False + ) -> None: """ Render a DataFrame to a LaTeX tabular/longtable environment output. """ @@ -831,7 +847,7 @@ def to_html( self, classes: Optional[Union[str, List, Tuple]] = None, notebook: bool = False, - border: Optional[int] = None, + border: Optional[int] = None ) -> None: """ Render a DataFrame to a html table. @@ -958,7 +974,7 @@ def _get_formatted_index(self, frame: "DataFrame") -> List[str]: return adjoined def _get_column_name_list(self) -> List[str]: - names = [] # type: List[str] + names: List[str] = [] columns = self.frame.columns if isinstance(columns, ABCMultiIndex): names.extend("" if name is None else name for name in columns.names) @@ -972,16 +988,16 @@ def _get_column_name_list(self) -> List[str]: def format_array( - values, - formatter, - float_format=None, - na_rep="NaN", - digits=None, - space=None, - justify="right", - decimal=".", - leading_space=None, -): + values: Any, + formatter: Optional[Union[Callable, Type[str]]], + float_format: Optional[Union[Callable, str, Type[str]]] = None, + na_rep: str = "NaN", + digits: Optional[int] = None, + space: Optional[Union[str, int]] = None, + justify: str = "right", + decimal: str = ".", + leading_space: Optional[bool] = None +) -> List[str]: """ Format an array for printing. @@ -1051,18 +1067,18 @@ def format_array( class GenericArrayFormatter: def __init__( self, - values, - digits=7, - formatter=None, - na_rep="NaN", - space=12, - float_format=None, - justify="right", - decimal=".", - quoting=None, - fixed_width=True, - leading_space=None, - ): + values: Any, + digits: int = 7, + formatter: Optional[Union[Callable, Type[str]]] = None, + na_rep: str = "NaN", + space: Union[str, int] = 12, + float_format: Optional[Union[str, Callable, EngFormatter, Type[str]]] = None, + justify: str = "right", + decimal: str = ".", + quoting: Optional[int] = None, + fixed_width: bool = True, + leading_space: Optional[bool] = None + ) -> None: self.values = values self.digits = digits self.na_rep = na_rep @@ -1075,11 +1091,11 @@ def __init__( self.fixed_width = fixed_width self.leading_space = leading_space - def get_result(self): + def get_result(self) -> Union[ndarray, List[str]]: fmt_values = self._format_strings() return _make_fixed_width(fmt_values, self.justify) - def _format_strings(self): + def _format_strings(self) -> List[str]: if self.float_format is None: float_format = get_option("display.float_format") if float_format is None: @@ -1149,7 +1165,7 @@ class FloatArrayFormatter(GenericArrayFormatter): """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: GenericArrayFormatter.__init__(self, *args, **kwargs) # float_format is expected to be a string @@ -1161,7 +1177,7 @@ def __init__(self, *args, **kwargs): self.formatter = self.float_format self.float_format = None - def _value_formatter(self, float_format=None, threshold=None): + def _value_formatter(self, float_format: Optional[Union[Callable, partial]] = None, threshold: Optional[Union[float, int]] = None) -> Callable: """Returns a function to be applied on each value to format it """ @@ -1207,7 +1223,7 @@ def formatter(value): return formatter - def get_result_as_array(self): + def get_result_as_array(self) -> Union[ndarray, List[str]]: """ Returns the float values converted into strings using the parameters given at initialisation, as a numpy array @@ -1296,7 +1312,7 @@ def format_values_with(float_format): return formatted_values - def _format_strings(self): + def _format_strings(self) -> List[str]: # shortcut if self.formatter is not None: return [self.formatter(x) for x in self.values] @@ -1305,19 +1321,19 @@ def _format_strings(self): class IntArrayFormatter(GenericArrayFormatter): - def _format_strings(self): + def _format_strings(self) -> List[str]: formatter = self.formatter or (lambda x: "{x: d}".format(x=x)) fmt_values = [formatter(x) for x in self.values] return fmt_values class Datetime64Formatter(GenericArrayFormatter): - def __init__(self, values, nat_rep="NaT", date_format=None, **kwargs): + def __init__(self, values: Union[ndarray, Series, DatetimeIndex, DatetimeArray], nat_rep: str = "NaT", date_format: None = None, **kwargs) -> None: super().__init__(values, **kwargs) self.nat_rep = nat_rep self.date_format = date_format - def _format_strings(self): + def _format_strings(self) -> List[str]: """ we by definition have DO NOT have a TZ """ values = self.values @@ -1337,7 +1353,7 @@ def _format_strings(self): class ExtensionArrayFormatter(GenericArrayFormatter): - def _format_strings(self): + def _format_strings(self) -> List[str]: values = self.values if isinstance(values, (ABCIndexClass, ABCSeries)): values = values._values @@ -1363,7 +1379,7 @@ def _format_strings(self): return fmt_values -def format_percentiles(percentiles): +def format_percentiles(percentiles: Union[ndarray, List[Union[int, float]], List[float], List[Union[str, float]]]) -> List[str]: """ Outputs rounded and formatted percentiles. @@ -1429,7 +1445,7 @@ def format_percentiles(percentiles): return [i + "%" for i in out] -def _is_dates_only(values): +def _is_dates_only(values: Union[ndarray, DatetimeArray, Index, DatetimeIndex]) -> bool: # return a boolean if we are only dates (and don't have a timezone) assert values.ndim == 1 @@ -1448,7 +1464,7 @@ def _is_dates_only(values): return False -def _format_datetime64(x, tz=None, nat_rep="NaT"): +def _format_datetime64(x: Union[NaTType, Timestamp], tz: Optional[Union[tzfile, tzutc]] = None, nat_rep: str = "NaT") -> str: if x is None or (is_scalar(x) and isna(x)): return nat_rep @@ -1461,7 +1477,7 @@ def _format_datetime64(x, tz=None, nat_rep="NaT"): return str(x) -def _format_datetime64_dateonly(x, nat_rep="NaT", date_format=None): +def _format_datetime64_dateonly(x: Union[NaTType, Timestamp], nat_rep: str = "NaT", date_format: None = None) -> str: if x is None or (is_scalar(x) and isna(x)): return nat_rep @@ -1474,7 +1490,7 @@ def _format_datetime64_dateonly(x, nat_rep="NaT", date_format=None): return x._date_repr -def _get_format_datetime64(is_dates_only, nat_rep="NaT", date_format=None): +def _get_format_datetime64(is_dates_only: bool, nat_rep: str = "NaT", date_format: None = None) -> Callable: if is_dates_only: return lambda x, tz=None: _format_datetime64_dateonly( @@ -1484,7 +1500,7 @@ def _get_format_datetime64(is_dates_only, nat_rep="NaT", date_format=None): return lambda x, tz=None: _format_datetime64(x, tz=tz, nat_rep=nat_rep) -def _get_format_datetime64_from_values(values, date_format): +def _get_format_datetime64_from_values(values: Union[ndarray, DatetimeArray, DatetimeIndex], date_format: Optional[str]) -> Optional[str]: """ given values and a date_format, return a string format """ if isinstance(values, np.ndarray) and values.ndim > 1: @@ -1499,7 +1515,7 @@ def _get_format_datetime64_from_values(values, date_format): class Datetime64TZFormatter(Datetime64Formatter): - def _format_strings(self): + def _format_strings(self) -> List[str]: """ we by definition have a TZ """ values = self.values.astype(object) @@ -1513,12 +1529,12 @@ def _format_strings(self): class Timedelta64Formatter(GenericArrayFormatter): - def __init__(self, values, nat_rep="NaT", box=False, **kwargs): + def __init__(self, values: Union[ndarray, TimedeltaIndex], nat_rep: str = "NaT", box: bool = False, **kwargs) -> None: super().__init__(values, **kwargs) self.nat_rep = nat_rep self.box = box - def _format_strings(self): + def _format_strings(self) -> ndarray: formatter = self.formatter or _get_format_timedelta64( self.values, nat_rep=self.nat_rep, box=self.box ) @@ -1526,7 +1542,7 @@ def _format_strings(self): return fmt_values -def _get_format_timedelta64(values, nat_rep="NaT", box=False): +def _get_format_timedelta64(values: Union[ndarray, TimedeltaIndex, TimedeltaArray], nat_rep: str = "NaT", box: bool = False) -> Callable: """ Return a formatter function for a range of timedeltas. These will all have the same format argument @@ -1567,7 +1583,7 @@ def _formatter(x): return _formatter -def _make_fixed_width(strings, justify="right", minimum=None, adj=None): +def _make_fixed_width(strings: Union[ndarray, List[str], List[str_]], justify: str = "right", minimum: Optional[int] = None, adj: Optional[Union[TextAdjustment, EastAsianTextAdjustment]] = None) -> Union[ndarray, List[str]]: if len(strings) == 0 or justify == "all": return strings @@ -1595,7 +1611,7 @@ def just(x): return result -def _trim_zeros_complex(str_complexes, na_rep="NaN"): +def _trim_zeros_complex(str_complexes: ndarray, na_rep: str = "NaN") -> List[str]: """ Separates the real and imaginary parts from the complex number, and executes the _trim_zeros_float method on each of those. @@ -1613,7 +1629,7 @@ def separate_and_trim(str_complex, na_rep): return ["".join(separate_and_trim(x, na_rep)) for x in str_complexes] -def _trim_zeros_float(str_floats, na_rep="NaN"): +def _trim_zeros_float(str_floats: Union[ndarray, List[str]], na_rep: str = "NaN") -> List[str]: """ Trims zeros, leaving just one before the decimal points if need be. """ @@ -1637,7 +1653,7 @@ def _cond(values): return [x + "0" if x.endswith(".") and _is_number(x) else x for x in trimmed] -def _has_names(index): +def _has_names(index: Index) -> bool: if isinstance(index, ABCMultiIndex): return com._any_not_none(*index.names) else: @@ -1672,11 +1688,11 @@ class EngFormatter: 24: "Y", } - def __init__(self, accuracy=None, use_eng_prefix=False): + def __init__(self, accuracy: Optional[int] = None, use_eng_prefix: bool = False) -> None: self.accuracy = accuracy self.use_eng_prefix = use_eng_prefix - def __call__(self, num): + def __call__(self, num: Union[float64, int, float]) -> str: """ Formats a number in engineering notation, appending a letter representing the power of 1000 of the original number. Some examples: @@ -1743,7 +1759,7 @@ def __call__(self, num): return formatted -def set_eng_float_format(accuracy=3, use_eng_prefix=False): +def set_eng_float_format(accuracy: int = 3, use_eng_prefix: bool = False) -> None: """ Alter default behavior on how float is formatted in DataFrame. Format float in engineering format. By accuracy, we mean the number of @@ -1756,7 +1772,7 @@ def set_eng_float_format(accuracy=3, use_eng_prefix=False): set_option("display.column_space", max(12, accuracy + 9)) -def _binify(cols, line_width): +def _binify(cols: List[int32], line_width: Union[int32, int]) -> List[int]: adjoin_width = 1 bins = [] curr_width = 0 @@ -1776,7 +1792,7 @@ def _binify(cols, line_width): return bins -def get_level_lengths(levels, sentinel=""): +def get_level_lengths(levels: Any, sentinel: Union[bool, object, str] = "") -> List[Dict[int, int]]: """For each index in each level the function returns lengths of indexes. Parameters @@ -1816,7 +1832,7 @@ def get_level_lengths(levels, sentinel=""): return result -def buffer_put_lines(buf, lines): +def buffer_put_lines(buf: Union[StringIO, TextIOWrapper, EncodedFile], lines: List[str]) -> None: """ Appends lines to a buffer. From cffc9356b5e02f64fcf6afe4b017acf376a696ea Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:03:12 +0100 Subject: [PATCH 02/35] isort and black --- pandas/io/formats/format.py | 149 ++++++++++++++++++++++++++---------- 1 file changed, 109 insertions(+), 40 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 7f77400cd2dcf..948d74079d0b3 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -4,18 +4,37 @@ """ from functools import partial -from io import StringIO +from io import StringIO, TextIOWrapper from shutil import get_terminal_size -from typing import TYPE_CHECKING, List, Optional, TextIO, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + TextIO, + Tuple, + Type, + Union, + cast, +) from unicodedata import east_asian_width +from _pytest.capture import EncodedFile +from dateutil.tz.tz import tzutc +from dateutil.zoneinfo import tzfile import numpy as np +from numpy import float64, int32, ndarray, str_ +from py._path.local import LocalPath from pandas._config.config import get_option, set_option from pandas._libs import lib from pandas._libs.tslib import format_array_from_datetime from pandas._libs.tslibs import NaT, Timedelta, Timestamp, iNaT +from pandas._libs.tslibs.nattype import NaTType +from pandas._libs.tslibs.timestamps import Timestamp from pandas.core.dtypes.common import ( is_categorical_dtype, @@ -40,30 +59,21 @@ ) from pandas.core.dtypes.missing import isna, notna +from pandas.core.arrays.datetimes import DatetimeArray +from pandas.core.arrays.timedeltas import TimedeltaArray from pandas.core.base import PandasObject import pandas.core.common as com +from pandas.core.frame import DataFrame from pandas.core.index import Index, ensure_index +from pandas.core.indexes.base import Index from pandas.core.indexes.datetimes import DatetimeIndex +from pandas.core.indexes.timedeltas import TimedeltaIndex +from pandas.core.series import Series +from pandas.tests.io.test_common import CustomFSPath from pandas.io.common import _expand_user, _stringify_path from pandas.io.formats.printing import adjoin, justify, pprint_thing -from io import StringIO, TextIOWrapper -from _pytest.capture import EncodedFile -from dateutil.tz.tz import tzutc -from dateutil.zoneinfo import tzfile -from numpy import float64, int32, ndarray, str_ -from pandas._libs.tslibs.nattype import NaTType -from pandas._libs.tslibs.timestamps import Timestamp -from pandas.core.arrays.datetimes import DatetimeArray -from pandas.core.arrays.timedeltas import TimedeltaArray -from pandas.core.frame import DataFrame -from pandas.core.indexes.base import Index -from pandas.core.indexes.timedeltas import TimedeltaIndex -from pandas.core.series import Series -from pandas.tests.io.test_common import CustomFSPath -from py._path.local import LocalPath -from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union if TYPE_CHECKING: from pandas import Series, DataFrame, Categorical @@ -153,7 +163,7 @@ def __init__( buf: Optional[TextIO] = None, length: bool = True, na_rep: str = "NaN", - footer: bool = True + footer: bool = True, ) -> None: self.categorical = categorical self.buf = buf if buf is not None else StringIO("") @@ -222,7 +232,7 @@ def __init__( float_format: Optional[str] = None, dtype: bool = True, max_rows: Optional[int] = None, - min_rows: Optional[int] = None + min_rows: Optional[int] = None, ) -> None: self.series = series self.buf = buf if buf is not None else StringIO() @@ -404,7 +414,17 @@ def len(self, text: str) -> int: self._EAW_MAP.get(east_asian_width(c), self.ambiguous_width) for c in text ) - def justify(self, texts: Union[Tuple[str, str], List[str], Tuple[str, str, str, str, str], Tuple[str, str, str, str]], max_len: int, mode: str = "right") -> List[str]: + def justify( + self, + texts: Union[ + Tuple[str, str], + List[str], + Tuple[str, str, str, str, str], + Tuple[str, str, str, str], + ], + max_len: int, + mode: str = "right", + ) -> List[str]: # re-calculate padding space per str considering East Asian Width def _get_pad(t): return max_len - self.len(t) + len(t) @@ -469,7 +489,9 @@ def __init__( header: Union[bool, List[str]] = True, index: Union[bool, int] = True, na_rep: str = "NaN", - formatters: Optional[Union[Tuple[Callable, Callable, Callable], Dict[str, Callable]]] = None, + formatters: Optional[ + Union[Tuple[Callable, Callable, Callable], Dict[str, Callable]] + ] = None, justify: Optional[str] = None, float_format: Optional[Union[str, Type[str]]] = None, sparsify: Optional[bool] = None, @@ -800,7 +822,7 @@ def to_latex( encoding: Optional[str] = None, multicolumn: bool = False, multicolumn_format: Optional[str] = None, - multirow: bool = False + multirow: bool = False, ) -> None: """ Render a DataFrame to a LaTeX tabular/longtable environment output. @@ -847,7 +869,7 @@ def to_html( self, classes: Optional[Union[str, List, Tuple]] = None, notebook: bool = False, - border: Optional[int] = None + border: Optional[int] = None, ) -> None: """ Render a DataFrame to a html table. @@ -996,7 +1018,7 @@ def format_array( space: Optional[Union[str, int]] = None, justify: str = "right", decimal: str = ".", - leading_space: Optional[bool] = None + leading_space: Optional[bool] = None, ) -> List[str]: """ Format an array for printing. @@ -1077,7 +1099,7 @@ def __init__( decimal: str = ".", quoting: Optional[int] = None, fixed_width: bool = True, - leading_space: Optional[bool] = None + leading_space: Optional[bool] = None, ) -> None: self.values = values self.digits = digits @@ -1177,7 +1199,11 @@ def __init__(self, *args, **kwargs) -> None: self.formatter = self.float_format self.float_format = None - def _value_formatter(self, float_format: Optional[Union[Callable, partial]] = None, threshold: Optional[Union[float, int]] = None) -> Callable: + def _value_formatter( + self, + float_format: Optional[Union[Callable, partial]] = None, + threshold: Optional[Union[float, int]] = None, + ) -> Callable: """Returns a function to be applied on each value to format it """ @@ -1328,7 +1354,13 @@ def _format_strings(self) -> List[str]: class Datetime64Formatter(GenericArrayFormatter): - def __init__(self, values: Union[ndarray, Series, DatetimeIndex, DatetimeArray], nat_rep: str = "NaT", date_format: None = None, **kwargs) -> None: + def __init__( + self, + values: Union[ndarray, Series, DatetimeIndex, DatetimeArray], + nat_rep: str = "NaT", + date_format: None = None, + **kwargs + ) -> None: super().__init__(values, **kwargs) self.nat_rep = nat_rep self.date_format = date_format @@ -1379,7 +1411,11 @@ def _format_strings(self) -> List[str]: return fmt_values -def format_percentiles(percentiles: Union[ndarray, List[Union[int, float]], List[float], List[Union[str, float]]]) -> List[str]: +def format_percentiles( + percentiles: Union[ + ndarray, List[Union[int, float]], List[float], List[Union[str, float]] + ] +) -> List[str]: """ Outputs rounded and formatted percentiles. @@ -1464,7 +1500,11 @@ def _is_dates_only(values: Union[ndarray, DatetimeArray, Index, DatetimeIndex]) return False -def _format_datetime64(x: Union[NaTType, Timestamp], tz: Optional[Union[tzfile, tzutc]] = None, nat_rep: str = "NaT") -> str: +def _format_datetime64( + x: Union[NaTType, Timestamp], + tz: Optional[Union[tzfile, tzutc]] = None, + nat_rep: str = "NaT", +) -> str: if x is None or (is_scalar(x) and isna(x)): return nat_rep @@ -1477,7 +1517,9 @@ def _format_datetime64(x: Union[NaTType, Timestamp], tz: Optional[Union[tzfile, return str(x) -def _format_datetime64_dateonly(x: Union[NaTType, Timestamp], nat_rep: str = "NaT", date_format: None = None) -> str: +def _format_datetime64_dateonly( + x: Union[NaTType, Timestamp], nat_rep: str = "NaT", date_format: None = None +) -> str: if x is None or (is_scalar(x) and isna(x)): return nat_rep @@ -1490,7 +1532,9 @@ def _format_datetime64_dateonly(x: Union[NaTType, Timestamp], nat_rep: str = "Na return x._date_repr -def _get_format_datetime64(is_dates_only: bool, nat_rep: str = "NaT", date_format: None = None) -> Callable: +def _get_format_datetime64( + is_dates_only: bool, nat_rep: str = "NaT", date_format: None = None +) -> Callable: if is_dates_only: return lambda x, tz=None: _format_datetime64_dateonly( @@ -1500,7 +1544,9 @@ def _get_format_datetime64(is_dates_only: bool, nat_rep: str = "NaT", date_forma return lambda x, tz=None: _format_datetime64(x, tz=tz, nat_rep=nat_rep) -def _get_format_datetime64_from_values(values: Union[ndarray, DatetimeArray, DatetimeIndex], date_format: Optional[str]) -> Optional[str]: +def _get_format_datetime64_from_values( + values: Union[ndarray, DatetimeArray, DatetimeIndex], date_format: Optional[str] +) -> Optional[str]: """ given values and a date_format, return a string format """ if isinstance(values, np.ndarray) and values.ndim > 1: @@ -1529,7 +1575,13 @@ def _format_strings(self) -> List[str]: class Timedelta64Formatter(GenericArrayFormatter): - def __init__(self, values: Union[ndarray, TimedeltaIndex], nat_rep: str = "NaT", box: bool = False, **kwargs) -> None: + def __init__( + self, + values: Union[ndarray, TimedeltaIndex], + nat_rep: str = "NaT", + box: bool = False, + **kwargs + ) -> None: super().__init__(values, **kwargs) self.nat_rep = nat_rep self.box = box @@ -1542,7 +1594,11 @@ def _format_strings(self) -> ndarray: return fmt_values -def _get_format_timedelta64(values: Union[ndarray, TimedeltaIndex, TimedeltaArray], nat_rep: str = "NaT", box: bool = False) -> Callable: +def _get_format_timedelta64( + values: Union[ndarray, TimedeltaIndex, TimedeltaArray], + nat_rep: str = "NaT", + box: bool = False, +) -> Callable: """ Return a formatter function for a range of timedeltas. These will all have the same format argument @@ -1583,7 +1639,12 @@ def _formatter(x): return _formatter -def _make_fixed_width(strings: Union[ndarray, List[str], List[str_]], justify: str = "right", minimum: Optional[int] = None, adj: Optional[Union[TextAdjustment, EastAsianTextAdjustment]] = None) -> Union[ndarray, List[str]]: +def _make_fixed_width( + strings: Union[ndarray, List[str], List[str_]], + justify: str = "right", + minimum: Optional[int] = None, + adj: Optional[Union[TextAdjustment, EastAsianTextAdjustment]] = None, +) -> Union[ndarray, List[str]]: if len(strings) == 0 or justify == "all": return strings @@ -1629,7 +1690,9 @@ def separate_and_trim(str_complex, na_rep): return ["".join(separate_and_trim(x, na_rep)) for x in str_complexes] -def _trim_zeros_float(str_floats: Union[ndarray, List[str]], na_rep: str = "NaN") -> List[str]: +def _trim_zeros_float( + str_floats: Union[ndarray, List[str]], na_rep: str = "NaN" +) -> List[str]: """ Trims zeros, leaving just one before the decimal points if need be. """ @@ -1688,7 +1751,9 @@ class EngFormatter: 24: "Y", } - def __init__(self, accuracy: Optional[int] = None, use_eng_prefix: bool = False) -> None: + def __init__( + self, accuracy: Optional[int] = None, use_eng_prefix: bool = False + ) -> None: self.accuracy = accuracy self.use_eng_prefix = use_eng_prefix @@ -1792,7 +1857,9 @@ def _binify(cols: List[int32], line_width: Union[int32, int]) -> List[int]: return bins -def get_level_lengths(levels: Any, sentinel: Union[bool, object, str] = "") -> List[Dict[int, int]]: +def get_level_lengths( + levels: Any, sentinel: Union[bool, object, str] = "" +) -> List[Dict[int, int]]: """For each index in each level the function returns lengths of indexes. Parameters @@ -1832,7 +1899,9 @@ def get_level_lengths(levels: Any, sentinel: Union[bool, object, str] = "") -> L return result -def buffer_put_lines(buf: Union[StringIO, TextIOWrapper, EncodedFile], lines: List[str]) -> None: +def buffer_put_lines( + buf: Union[StringIO, TextIOWrapper, EncodedFile], lines: List[str] +) -> None: """ Appends lines to a buffer. From 5c89d1538ecffb15bf36243f0511a9b905262bd0 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:13:23 +0100 Subject: [PATCH 03/35] redefinition of unused 'Timestamp' from line 35flake8(F811) --- pandas/io/formats/format.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 948d74079d0b3..a9a4ac4e6a7c9 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -34,7 +34,6 @@ from pandas._libs.tslib import format_array_from_datetime from pandas._libs.tslibs import NaT, Timedelta, Timestamp, iNaT from pandas._libs.tslibs.nattype import NaTType -from pandas._libs.tslibs.timestamps import Timestamp from pandas.core.dtypes.common import ( is_categorical_dtype, From 033846e077f5c9e43d5aa7b548a0335075aa1ada Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:14:46 +0100 Subject: [PATCH 04/35] redefinition of unused 'Index' from line 66flake8(F811) --- pandas/io/formats/format.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index a9a4ac4e6a7c9..794900a6b104c 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -64,7 +64,6 @@ import pandas.core.common as com from pandas.core.frame import DataFrame from pandas.core.index import Index, ensure_index -from pandas.core.indexes.base import Index from pandas.core.indexes.datetimes import DatetimeIndex from pandas.core.indexes.timedeltas import TimedeltaIndex from pandas.core.series import Series From 062c0bbcc1fd43f304409652992cf9f440ff52cc Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:16:04 +0100 Subject: [PATCH 05/35] undefined name 'EngFormatter'flake8(F821) --- pandas/io/formats/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 794900a6b104c..031dbd1779835 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -1092,7 +1092,7 @@ def __init__( formatter: Optional[Union[Callable, Type[str]]] = None, na_rep: str = "NaN", space: Union[str, int] = 12, - float_format: Optional[Union[str, Callable, EngFormatter, Type[str]]] = None, + float_format: Optional[Union[str, Callable, "EngFormatter", Type[str]]] = None, justify: str = "right", decimal: str = ".", quoting: Optional[int] = None, From 3f4f1ce14ab37cc7e7481b1b5320c78ef056c483 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:16:59 +0100 Subject: [PATCH 06/35] redefinition of unused 'Series' from line 69flake8(F811) --- pandas/io/formats/format.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 031dbd1779835..05dfdbe8e0592 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -66,7 +66,6 @@ from pandas.core.index import Index, ensure_index from pandas.core.indexes.datetimes import DatetimeIndex from pandas.core.indexes.timedeltas import TimedeltaIndex -from pandas.core.series import Series from pandas.tests.io.test_common import CustomFSPath from pandas.io.common import _expand_user, _stringify_path From fc06ebbf1675e995b0248541545afe92fd1040de Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:17:20 +0100 Subject: [PATCH 07/35] redefinition of unused 'DataFrame' from line 65flake8(F811) --- pandas/io/formats/format.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 05dfdbe8e0592..3eb2b518fde6a 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -62,7 +62,6 @@ from pandas.core.arrays.timedeltas import TimedeltaArray from pandas.core.base import PandasObject import pandas.core.common as com -from pandas.core.frame import DataFrame from pandas.core.index import Index, ensure_index from pandas.core.indexes.datetimes import DatetimeIndex from pandas.core.indexes.timedeltas import TimedeltaIndex From 352d57cacad319eb6fd6d9b83d9fb75b2a308ad1 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:25:08 +0100 Subject: [PATCH 08/35] ImportError: cannot import name 'Categorical' from 'pandas' --- pandas/io/formats/format.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 3eb2b518fde6a..1fd3d253d11f9 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -65,7 +65,6 @@ from pandas.core.index import Index, ensure_index from pandas.core.indexes.datetimes import DatetimeIndex from pandas.core.indexes.timedeltas import TimedeltaIndex -from pandas.tests.io.test_common import CustomFSPath from pandas.io.common import _expand_user, _stringify_path from pandas.io.formats.printing import adjoin, justify, pprint_thing @@ -479,7 +478,7 @@ class DataFrameFormatter(TableFormatter): def __init__( self, frame: DataFrame, - buf: Optional[Union[StringIO, LocalPath, str, CustomFSPath]] = None, + buf: Optional[Union[StringIO, LocalPath, str]] = None, columns: Optional[List[str]] = None, col_space: Optional[Union[str, int]] = None, header: Union[bool, List[str]] = True, From 8b1c63c6061ab2753a1f754965ff2cc3b1115a98 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:26:12 +0100 Subject: [PATCH 09/35] NameError: name 'DataFrame' is not defined --- pandas/io/formats/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 1fd3d253d11f9..fd6e2f048a1f4 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -477,7 +477,7 @@ class DataFrameFormatter(TableFormatter): def __init__( self, - frame: DataFrame, + frame: "DataFrame", buf: Optional[Union[StringIO, LocalPath, str]] = None, columns: Optional[List[str]] = None, col_space: Optional[Union[str, int]] = None, From d601eada5759fb162cb2445af439bc57fb85cd12 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:27:33 +0100 Subject: [PATCH 10/35] NameError: name 'Series' is not defined --- pandas/io/formats/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index fd6e2f048a1f4..c4dd0e3612346 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -1351,7 +1351,7 @@ def _format_strings(self) -> List[str]: class Datetime64Formatter(GenericArrayFormatter): def __init__( self, - values: Union[ndarray, Series, DatetimeIndex, DatetimeArray], + values: Union[ndarray, "Series", DatetimeIndex, DatetimeArray], nat_rep: str = "NaT", date_format: None = None, **kwargs From 2a97b7606bdabd39f58c72eb684a2b053fa4fceb Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:39:34 +0100 Subject: [PATCH 11/35] tests passing locally From bf8706dcb91f3c415a2be1a8f8b2a9d7bbb7161b Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:46:09 +0100 Subject: [PATCH 12/35] undo unwanted changes to py3.5 Syntax for Variable Annotations --- pandas/io/formats/format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index c4dd0e3612346..3ca35ac2eec0d 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -268,7 +268,7 @@ def _chk_truncate(self) -> None: else: row_num = max_rows // 2 series = concat((series.iloc[:row_num], series.iloc[-row_num:])) - self.tr_row_num: Optional[int] = row_num + self.tr_row_num = row_num # type: Optional[int] else: self.tr_row_num = None self.tr_series = series @@ -991,7 +991,7 @@ def _get_formatted_index(self, frame: "DataFrame") -> List[str]: return adjoined def _get_column_name_list(self) -> List[str]: - names: List[str] = [] + names = [] # type: List[str] columns = self.frame.columns if isinstance(columns, ABCMultiIndex): names.extend("" if name is None else name for name in columns.names) From c02a0fae16eb0b4e068429af70ff5a18425ee782 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 20:52:05 +0100 Subject: [PATCH 13/35] error: "TableFormatter" has no attribute ... --- pandas/io/formats/format.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 3ca35ac2eec0d..3d047e31f59b6 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -443,6 +443,9 @@ def _get_adjustment() -> Union[TextAdjustment, EastAsianTextAdjustment]: class TableFormatter: show_dimensions = None + is_truncated = None + formatters = None + columns = None @property def should_show_dimensions(self) -> Optional[bool]: From 10d0fa3f42b0320b3ffd6e631972ceac7de9c027 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 21:11:06 +0100 Subject: [PATCH 14/35] pandas\io\formats\format.py:463: error: Unsupported right operand type for in ("None") --- pandas/io/formats/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 3d047e31f59b6..ab773632735ce 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -445,7 +445,7 @@ class TableFormatter: show_dimensions = None is_truncated = None formatters = None - columns = None + columns = None # type: Index @property def should_show_dimensions(self) -> Optional[bool]: From 035249dc3d2d62b63d9fce9b5dd7b23fdb201f52 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 21:34:38 +0100 Subject: [PATCH 15/35] pandas\io\formats\format.py:465: error: "None" has no attribute "get" --- pandas/io/formats/format.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index ab773632735ce..d72d7a1cfcd96 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -72,6 +72,10 @@ if TYPE_CHECKING: from pandas import Series, DataFrame, Categorical +formatters_type = Union[ + List[Callable], Tuple[Callable, ...], Dict[Union[str, int], Callable] +] + common_docstring = """ Parameters ---------- @@ -87,7 +91,7 @@ Whether to print index (row) labels. na_rep : str, optional, default 'NaN' String representation of NAN to use. - formatters : list or dict of one-param. functions, optional + formatters : list, tuple or dict of one-param. functions, optional Formatter functions to apply to columns' elements by position or name. The result of each function must be a unicode string. @@ -444,7 +448,7 @@ class TableFormatter: show_dimensions = None is_truncated = None - formatters = None + formatters = None # type: formatters_type columns = None # type: Index @property @@ -456,6 +460,7 @@ def should_show_dimensions(self) -> Optional[bool]: def _get_formatter(self, i: Union[str, int]) -> Optional[Callable]: if isinstance(self.formatters, (list, tuple)): if is_integer(i): + i = cast(int, i) return self.formatters[i] else: return None @@ -487,9 +492,7 @@ def __init__( header: Union[bool, List[str]] = True, index: Union[bool, int] = True, na_rep: str = "NaN", - formatters: Optional[ - Union[Tuple[Callable, Callable, Callable], Dict[str, Callable]] - ] = None, + formatters: Optional[formatters_type] = None, justify: Optional[str] = None, float_format: Optional[Union[str, Type[str]]] = None, sparsify: Optional[bool] = None, From fbaa22482c2cf213c2fe5e2d4421cb275ea39c01 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 21:48:45 +0100 Subject: [PATCH 16/35] pandas\io\formats\format.py:534: error: Incompatible types in assignment (expression has type "Union[bool, str]", variable has type "None") --- pandas/io/formats/format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index d72d7a1cfcd96..cd2d7f3751f76 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -446,7 +446,7 @@ def _get_adjustment() -> Union[TextAdjustment, EastAsianTextAdjustment]: class TableFormatter: - show_dimensions = None + show_dimensions = None # type: bool is_truncated = None formatters = None # type: formatters_type columns = None # type: Index @@ -501,7 +501,7 @@ def __init__( max_rows: Optional[int] = None, min_rows: Optional[int] = None, max_cols: Optional[int] = None, - show_dimensions: Union[bool, str] = False, + show_dimensions: bool = False, decimal: str = ".", table_id: Optional[str] = None, render_links: bool = False, From e955a52b470162d0b42149c63e65ba0ecefa1692 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 21:59:04 +0100 Subject: [PATCH 17/35] pandas\io\formats\format.py:574: error: Unsupported operand types for + ("List[str]" and "int") --- pandas/io/formats/format.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index cd2d7f3751f76..fd4457c625acb 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -571,6 +571,9 @@ def _chk_truncate(self) -> None: prompt_row = 1 if self.show_dimensions: show_dimension_rows = 3 + # assume we only get here if self.header is boolean. + # i.e. not to_latex() where self.header may be List[str] + self.header = cast(bool, self.header) n_add_rows = self.header + dot_row + show_dimension_rows + prompt_row # rows available to fill with actual data max_rows_adj = self.h - n_add_rows From 32de27bd26fe251d90b271b0993b14d30c56b3ee Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 22:11:03 +0100 Subject: [PATCH 18/35] pandas\io\formats\format.py:594: error: Incompatible types in assignment (expression has type "Optional[int]", variable has type "int") --- pandas/io/formats/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index fd4457c625acb..c06c120738e93 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -576,7 +576,7 @@ def _chk_truncate(self) -> None: self.header = cast(bool, self.header) n_add_rows = self.header + dot_row + show_dimension_rows + prompt_row # rows available to fill with actual data - max_rows_adj = self.h - n_add_rows + max_rows_adj = self.h - n_add_rows # type: Optional[int] self.max_rows_adj = max_rows_adj # Format only rows and columns that could potentially fit the From 5802c0e1d33e4ba567c2498884d8d239517f36c6 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 22:22:11 +0100 Subject: [PATCH 19/35] pandas\io\formats\format.py:610: error: Incompatible types in assignment (expression has type "Optional[int]", variable has type "int") --- pandas/io/formats/format.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index c06c120738e93..dbf1d3683e07a 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -603,9 +603,12 @@ def _chk_truncate(self) -> None: frame = self.frame if truncate_h: + # cast here since if truncate_h is True, max_cols_adj is not None + max_cols_adj = cast(int, max_cols_adj) if max_cols_adj == 0: col_num = len(frame.columns) elif max_cols_adj == 1: + max_cols = cast(int, max_cols) frame = frame.iloc[:, :max_cols] col_num = max_cols else: From b82a952ca7d13e7f80929571518056e11010af2a Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 22:24:20 +0100 Subject: [PATCH 20/35] pandas\io\formats\format.py:625: error: Unsupported operand types for // ("None" and "int") --- pandas/io/formats/format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index dbf1d3683e07a..262f339000f3b 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -618,6 +618,8 @@ def _chk_truncate(self) -> None: ) self.tr_col_num = col_num if truncate_v: + # cast here since if truncate_v is True, max_rows_adj is not None + max_rows_adj = cast(int, max_rows_adj) if max_rows_adj == 1: row_num = max_rows frame = frame.iloc[:max_rows, :] From 31080c7a706fa898a86aa4e5e9faf0b78a791a6c Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 22:29:07 +0100 Subject: [PATCH 21/35] pandas\io\formats\format.py:636: error: Incompatible types in assignment (expression has type "Optional[int]", variable has type "None") --- pandas/io/formats/format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 262f339000f3b..1f794bdaebaee 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -447,7 +447,7 @@ def _get_adjustment() -> Union[TextAdjustment, EastAsianTextAdjustment]: class TableFormatter: show_dimensions = None # type: bool - is_truncated = None + is_truncated = None # type: bool formatters = None # type: formatters_type columns = None # type: Index @@ -633,7 +633,7 @@ def _chk_truncate(self) -> None: self.tr_frame = frame self.truncate_h = truncate_h self.truncate_v = truncate_v - self.is_truncated = self.truncate_h or self.truncate_v + self.is_truncated = bool(self.truncate_h or self.truncate_v) def _to_str_columns(self) -> List[List[str]]: """ From f0dc771a43d1fb509ad26353d547979ae45f5f01 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 23:20:49 +0100 Subject: [PATCH 22/35] pandas\io\formats\format.py:651: error: Argument "minimum" to "_make_fixed_width" has incompatible type "Union[str, int]"; expected "Optional[int]" --- pandas/io/formats/format.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 1f794bdaebaee..c9cff96f26c38 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -95,7 +95,7 @@ Formatter functions to apply to columns' elements by position or name. The result of each function must be a unicode string. - List must be of length equal to the number of columns. + List/tuple must be of length equal to the number of columns. float_format : one-parameter function, optional, default None Formatter function to apply to columns' elements if they are floats. The result of this function must be a unicode string. @@ -639,6 +639,10 @@ def _to_str_columns(self) -> List[List[str]]: """ Render a DataFrame to a list of columns (as lists of strings). """ + # this method is not used by to_html where self.col_space + # could be a string so safe to cast + self.col_space = cast(int, self.col_space) + frame = self.tr_frame # may include levels names also From bd7887cd25e7bda30527b6951384a02ea9ddf619 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 23:27:14 +0100 Subject: [PATCH 23/35] pandas\io\formats\format.py:664: error: Argument 1 to "len" has incompatible type "Union[bool, List[str]]"; expected "Sized" --- pandas/io/formats/format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index c9cff96f26c38..36ff5b8cee522 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -661,6 +661,8 @@ def _to_str_columns(self) -> List[List[str]]: stringified.append(fmt_values) else: if is_list_like(self.header): + # cast here since can't be bool if is_list_like + self.header = cast(List[str], self.header) if len(self.header) != len(self.columns): raise ValueError( ( From cfc1806f2bb093f81cd764e5499f65c9f3fcee68 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 23:37:46 +0100 Subject: [PATCH 24/35] pandas\io\formats\format.py:714: error: Invalid index type "Optional[int]" for "Union[Any, List[str]]"; expected type "int" --- pandas/io/formats/format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 36ff5b8cee522..d31a3addc369d 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -709,6 +709,8 @@ def _to_str_columns(self) -> List[List[str]]: if truncate_v: n_header_rows = len(str_index) - len(frame) row_num = self.tr_row_num + # cast here since if truncate_v is True, self.tr_row_num is not None + row_num = cast(int, row_num) for ix, col in enumerate(strcols): # infer from above row cwidth = self.adj.len(strcols[ix][row_num]) From 2a429f64b8ef45d289d124bae3a20ffb6d88ef34 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 23:43:03 +0100 Subject: [PATCH 25/35] pandas\io\formats\format.py:762: error: Incompatible types in assignment (expression has type "List[str]", variable has type "str") --- pandas/io/formats/format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index d31a3addc369d..463dcab65ac39 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -759,8 +759,8 @@ def to_string(self) -> None: ): # need to wrap around text = self._join_multiline(*strcols) else: # max_cols == 0. Try to fit frame to terminal - text = self.adj.adjoin(1, *strcols).split("\n") - max_len = Series(text).str.len().max() + lines = self.adj.adjoin(1, *strcols).split("\n") + max_len = Series(lines).str.len().max() # plus truncate dot col dif = max_len - self.w # '+ 1' to avoid too wide repr (GH PR #17023) From 02c364a8e9bd98d89cd1730e4cc551d882cfc5a9 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 23:44:03 +0100 Subject: [PATCH 26/35] pandas\io\formats\format.py:803: error: Incompatible types in assignment (expression has type "List[Any]", variable has type "Tuple[Any, ...]") --- pandas/io/formats/format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 463dcab65ac39..fca5132fd466e 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -797,10 +797,10 @@ def to_string(self) -> None: ) ) - def _join_multiline(self, *strcols) -> str: + def _join_multiline(self, *args) -> str: lwidth = self.line_width adjoin_width = 1 - strcols = list(strcols) + strcols = list(args) if self.index: idx = strcols.pop(0) lwidth -= np.array([self.adj.len(x) for x in idx]).max() + adjoin_width From 9d2a734c2937afc6054a28c2dc05ec2c3852f20b Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 23:46:51 +0100 Subject: [PATCH 27/35] pandas\io\formats\format.py:816: error: Unsupported operand types for + ("None" and "int") --- pandas/io/formats/format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index fca5132fd466e..48ab1f7abd4e9 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -813,6 +813,8 @@ def _join_multiline(self, *args) -> str: nbins = len(col_bins) if self.truncate_v: + # cast here since if truncate_v is True, max_rows_adj is not None + self.max_rows_adj = cast(int, self.max_rows_adj) nrows = self.max_rows_adj + 1 else: nrows = len(self.frame) From a70080b36845b02f0f6ac0c6927434b3c55c83dd Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 23:50:13 +0100 Subject: [PATCH 28/35] pandas\io\formats\format.py:997: error: Argument "minimum" to "_make_fixed_width" has incompatible type "Union[str, int]"; expected "Optional[int]" --- pandas/io/formats/format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 48ab1f7abd4e9..270c74a236ff6 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -977,7 +977,8 @@ def show_col_idx_names(self) -> bool: def _get_formatted_index(self, frame: "DataFrame") -> List[str]: # Note: this is only used by to_string() and to_latex(), not by - # to_html(). + # to_html(). so safe to cast col_space here. + self.col_space = cast(int, self.col_space) index = frame.index columns = frame.columns fmt = self._get_formatter("__index__") From 5d2c6806048d7373ce73c13c6aba2e4ae7170015 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sun, 21 Jul 2019 23:53:16 +0100 Subject: [PATCH 29/35] pandas\io\formats\format.py:1075: error: Incompatible types in assignment (expression has type "Type[Timedelta64Formatter]", variable has type "Type[Datetime64Formatter]") --- pandas/io/formats/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 270c74a236ff6..2281992d5350a 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -1068,7 +1068,7 @@ def format_array( """ if is_datetime64_dtype(values.dtype): - fmt_klass = Datetime64Formatter + fmt_klass = Datetime64Formatter # type: Type[GenericArrayFormatter] elif is_datetime64tz_dtype(values): fmt_klass = Datetime64TZFormatter elif is_timedelta64_dtype(values.dtype): From 660136a94a9a154168bdf1593b4089e590b938b9 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 22 Jul 2019 00:04:46 +0100 Subject: [PATCH 30/35] pandas\io\formats\format.py:1232: error: Incompatible types in assignment (expression has type "Union[str, Callable[..., Any], EngFormatter, Type[str], None]", variable has type "Union[Callable[..., Any], partial[Any], None]") --- pandas/io/formats/format.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 2281992d5350a..3fdae18508a58 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -75,6 +75,7 @@ formatters_type = Union[ List[Callable], Tuple[Callable, ...], Dict[Union[str, int], Callable] ] +float_format_type = Union[str, Callable[..., Any], "EngFormatter", Type[str]] common_docstring = """ Parameters @@ -1114,7 +1115,7 @@ def __init__( formatter: Optional[Union[Callable, Type[str]]] = None, na_rep: str = "NaN", space: Union[str, int] = 12, - float_format: Optional[Union[str, Callable, "EngFormatter", Type[str]]] = None, + float_format: Optional[float_format_type] = None, justify: str = "right", decimal: str = ".", quoting: Optional[int] = None, @@ -1221,7 +1222,7 @@ def __init__(self, *args, **kwargs) -> None: def _value_formatter( self, - float_format: Optional[Union[Callable, partial]] = None, + float_format: Optional[float_format_type] = None, threshold: Optional[Union[float, int]] = None, ) -> Callable: """Returns a function to be applied on each value to format it From 80922a38b28854b5da3d97d555bf2eb922d328dd Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 22 Jul 2019 00:08:43 +0100 Subject: [PATCH 31/35] pandas\io\formats\format.py:1326: error: Incompatible types in assignment (expression has type "None", variable has type "partial[str]") --- pandas/io/formats/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index 3fdae18508a58..d307afad0e0c7 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -1322,7 +1322,7 @@ def format_values_with(float_format): if self.fixed_width: float_format = partial( "{value: .{digits:d}f}".format, digits=self.digits - ) + ) # type: Optional[float_format_type] else: float_format = self.float_format else: From 4aae208b5461bd734e3ea76ce0b1fa6a907044b3 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 22 Jul 2019 00:42:38 +0100 Subject: [PATCH 32/35] pandas\io\formats\html.py:86: error: Signature of "is_truncated" incompatible with supertype "TableFormatter" --- pandas/io/formats/html.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/io/formats/html.py b/pandas/io/formats/html.py index 91e90a78d87a7..56c58a1b1363f 100644 --- a/pandas/io/formats/html.py +++ b/pandas/io/formats/html.py @@ -82,8 +82,9 @@ def row_levels(self) -> int: def _get_columns_formatted_values(self) -> Iterable: return self.columns + # https://github.com/python/mypy/issues/1237 @property - def is_truncated(self) -> bool: + def is_truncated(self) -> bool: # type: ignore return self.fmt.is_truncated @property From 8fa7b2912b60dccefac1cf61d8a98346e147ce58 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 22 Jul 2019 00:47:14 +0100 Subject: [PATCH 33/35] pandas\io\formats\html.py:466: error: Unsupported operand types for >= ("int" and "None") --- pandas/io/formats/html.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/io/formats/html.py b/pandas/io/formats/html.py index 56c58a1b1363f..19305126f4e5f 100644 --- a/pandas/io/formats/html.py +++ b/pandas/io/formats/html.py @@ -4,7 +4,7 @@ from collections import OrderedDict from textwrap import dedent -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, cast from pandas._config import get_option @@ -459,6 +459,8 @@ def _write_hierarchical_rows( # Insert ... row and adjust idx_values and # level_lengths to take this into account. ins_row = self.fmt.tr_row_num + # cast here since if truncate_v is True, self.fmt.tr_row_num is not None + ins_row = cast(int, ins_row) inserted = False for lnum, records in enumerate(level_lengths): rec_new = {} From bb15c301b56630f9748a5d7273a5cbc0125bfbc5 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 22 Jul 2019 00:52:26 +0100 Subject: [PATCH 34/35] remove None return types from __init__ (automatically reapplied by monkeytype) --- pandas/io/formats/format.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index d307afad0e0c7..f2c967cce21ae 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -164,7 +164,7 @@ def __init__( length: bool = True, na_rep: str = "NaN", footer: bool = True, - ) -> None: + ): self.categorical = categorical self.buf = buf if buf is not None else StringIO("") self.na_rep = na_rep @@ -233,7 +233,7 @@ def __init__( dtype: bool = True, max_rows: Optional[int] = None, min_rows: Optional[int] = None, - ) -> None: + ): self.series = series self.buf = buf if buf is not None else StringIO() self.name = name @@ -377,7 +377,7 @@ def to_string(self) -> str: class TextAdjustment: - def __init__(self) -> None: + def __init__(self): self.encoding = get_option("display.encoding") def len(self, text: Union[str_, str]) -> int: @@ -391,7 +391,7 @@ def adjoin(self, space: int, *lists, **kwargs) -> str: class EastAsianTextAdjustment(TextAdjustment): - def __init__(self) -> None: + def __init__(self): super().__init__() if get_option("display.unicode.ambiguous_as_wide"): self.ambiguous_width = 2 @@ -507,7 +507,7 @@ def __init__( table_id: Optional[str] = None, render_links: bool = False, **kwds - ) -> None: + ): self.frame = frame if buf is not None: self.buf = _expand_user(_stringify_path(buf)) @@ -1121,7 +1121,7 @@ def __init__( quoting: Optional[int] = None, fixed_width: bool = True, leading_space: Optional[bool] = None, - ) -> None: + ): self.values = values self.digits = digits self.na_rep = na_rep @@ -1208,7 +1208,7 @@ class FloatArrayFormatter(GenericArrayFormatter): """ - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args, **kwargs): GenericArrayFormatter.__init__(self, *args, **kwargs) # float_format is expected to be a string @@ -1381,7 +1381,7 @@ def __init__( nat_rep: str = "NaT", date_format: None = None, **kwargs - ) -> None: + ): super().__init__(values, **kwargs) self.nat_rep = nat_rep self.date_format = date_format @@ -1602,7 +1602,7 @@ def __init__( nat_rep: str = "NaT", box: bool = False, **kwargs - ) -> None: + ): super().__init__(values, **kwargs) self.nat_rep = nat_rep self.box = box @@ -1772,9 +1772,7 @@ class EngFormatter: 24: "Y", } - def __init__( - self, accuracy: Optional[int] = None, use_eng_prefix: bool = False - ) -> None: + def __init__(self, accuracy: Optional[int] = None, use_eng_prefix: bool = False): self.accuracy = accuracy self.use_eng_prefix = use_eng_prefix From 63aa725e1a80354aa26bdd1a210d7bee18ab424b Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 22 Jul 2019 01:59:21 +0100 Subject: [PATCH 35/35] simplify some unions --- pandas/io/formats/format.py | 44 ++++++++++++++----------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index f2c967cce21ae..c5b0b7b222e27 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -4,13 +4,14 @@ """ from functools import partial -from io import StringIO, TextIOWrapper +from io import StringIO from shutil import get_terminal_size from typing import ( TYPE_CHECKING, Any, Callable, Dict, + Iterable, List, Optional, TextIO, @@ -21,12 +22,10 @@ ) from unicodedata import east_asian_width -from _pytest.capture import EncodedFile from dateutil.tz.tz import tzutc from dateutil.zoneinfo import tzfile import numpy as np -from numpy import float64, int32, ndarray, str_ -from py._path.local import LocalPath +from numpy import float64, int32, ndarray from pandas._config.config import get_option, set_option @@ -58,6 +57,7 @@ ) from pandas.core.dtypes.missing import isna, notna +from pandas._typing import FilePathOrBuffer from pandas.core.arrays.datetimes import DatetimeArray from pandas.core.arrays.timedeltas import TimedeltaArray from pandas.core.base import PandasObject @@ -75,7 +75,7 @@ formatters_type = Union[ List[Callable], Tuple[Callable, ...], Dict[Union[str, int], Callable] ] -float_format_type = Union[str, Callable[..., Any], "EngFormatter", Type[str]] +float_format_type = Union[str, Callable, "EngFormatter"] common_docstring = """ Parameters @@ -380,7 +380,7 @@ class TextAdjustment: def __init__(self): self.encoding = get_option("display.encoding") - def len(self, text: Union[str_, str]) -> int: + def len(self, text: str) -> int: return len(text) def justify(self, texts: Any, max_len: int, mode: str = "right") -> List[str]: @@ -415,15 +415,7 @@ def len(self, text: str) -> int: ) def justify( - self, - texts: Union[ - Tuple[str, str], - List[str], - Tuple[str, str, str, str, str], - Tuple[str, str, str, str], - ], - max_len: int, - mode: str = "right", + self, texts: Iterable[str], max_len: int, mode: str = "right" ) -> List[str]: # re-calculate padding space per str considering East Asian Width def _get_pad(t): @@ -437,7 +429,7 @@ def _get_pad(t): return [x.rjust(_get_pad(x)) for x in texts] -def _get_adjustment() -> Union[TextAdjustment, EastAsianTextAdjustment]: +def _get_adjustment() -> TextAdjustment: use_east_asian_width = get_option("display.unicode.east_asian_width") if use_east_asian_width: return EastAsianTextAdjustment() @@ -487,15 +479,15 @@ class DataFrameFormatter(TableFormatter): def __init__( self, frame: "DataFrame", - buf: Optional[Union[StringIO, LocalPath, str]] = None, + buf: Optional[FilePathOrBuffer] = None, columns: Optional[List[str]] = None, col_space: Optional[Union[str, int]] = None, header: Union[bool, List[str]] = True, - index: Union[bool, int] = True, + index: bool = True, na_rep: str = "NaN", formatters: Optional[formatters_type] = None, justify: Optional[str] = None, - float_format: Optional[Union[str, Type[str]]] = None, + float_format: Optional[float_format_type] = None, sparsify: Optional[bool] = None, index_names: bool = True, line_width: Optional[int] = None, @@ -1032,8 +1024,8 @@ def _get_column_name_list(self) -> List[str]: def format_array( values: Any, - formatter: Optional[Union[Callable, Type[str]]], - float_format: Optional[Union[Callable, str, Type[str]]] = None, + formatter: Optional[Callable], + float_format: Optional[float_format_type] = None, na_rep: str = "NaN", digits: Optional[int] = None, space: Optional[Union[str, int]] = None, @@ -1112,7 +1104,7 @@ def __init__( self, values: Any, digits: int = 7, - formatter: Optional[Union[Callable, Type[str]]] = None, + formatter: Optional[Callable] = None, na_rep: str = "NaN", space: Union[str, int] = 12, float_format: Optional[float_format_type] = None, @@ -1661,10 +1653,10 @@ def _formatter(x): def _make_fixed_width( - strings: Union[ndarray, List[str], List[str_]], + strings: Union[ndarray, List[str]], justify: str = "right", minimum: Optional[int] = None, - adj: Optional[Union[TextAdjustment, EastAsianTextAdjustment]] = None, + adj: Optional[TextAdjustment] = None, ) -> Union[ndarray, List[str]]: if len(strings) == 0 or justify == "all": @@ -1918,9 +1910,7 @@ def get_level_lengths( return result -def buffer_put_lines( - buf: Union[StringIO, TextIOWrapper, EncodedFile], lines: List[str] -) -> None: +def buffer_put_lines(buf: TextIO, lines: List[str]) -> None: """ Appends lines to a buffer.