Skip to content

CLN: remove util._decorators.make_signature and make related changes #26819

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 14, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion pandas/_typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from pandas.core.dtypes.dtypes import ExtensionDtype
from pandas.core.dtypes.generic import (
ABCExtensionArray, ABCIndexClass, ABCSeries, ABCSparseSeries)
ABCDataFrame, ABCExtensionArray, ABCIndexClass, ABCSeries, ABCSparseSeries)

AnyArrayLike = TypeVar('AnyArrayLike',
ABCExtensionArray,
Expand All @@ -22,3 +22,5 @@
Timedelta)
Dtype = Union[str, np.dtype, ExtensionDtype]
FilePathOrBuffer = Union[str, Path, IO[AnyStr]]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@WillAyd we should put a line comment on when to use these / what these are if not completely obvious (and then maybe even so).

future PR of course

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea I'm going to take another look at this module soon to see if there's a better way of doing things. Also wondering if we should use if TYPE_CHECKING imports in that module instead of the ABCs. Will check it out and post separately

Copy link
Contributor

@jreback jreback Jun 14, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea I'm going to take another look at this module soon to see if there's a better way of doing things. Also wondering if we should use if TYPE_CHECKING imports in that module instead of the ABCs. Will check it out and post separately

-1 on using TYPE_CHECCKING. I think the way you have it setup is very nice actually, unless I am missing something big.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My concern is that the ABC* classes aren't getting type checked correctly. Adding reveal_type(klass) to this PR where the TypeVar is used yields the following:

$ mypy pandas/core/groupby/generic.py
pandas/core/groupby/generic.py:83: error: Revealed type is 'Type[Any]'

That coupled with @topper-123 's comment around code completion makes me think the ABCs might all be Type[Any] since they are generated by create_pandas_abc_type, a function without any annotations itself (so implicitly Type[Any])

Going to investigate and open a separate issue accordingly

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok fair. I just don't want to introduce overhead for contributors, which typing especially with 'non-standard' things (e.g. cast) can do; we really want to minimize what people have to do (nothwithstanding maybe we have magic that makes things easy).


FrameOrSeries = TypeVar('FrameOrSeries', ABCSeries, ABCDataFrame)
69 changes: 0 additions & 69 deletions pandas/core/groupby/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
hold the whitelist of methods that are exposed on the
SeriesGroupBy and the DataFrameGroupBy objects.
"""

import types

from pandas.util._decorators import make_signature

from pandas.core.dtypes.common import is_list_like, is_scalar


Expand Down Expand Up @@ -91,67 +86,3 @@ def _gotitem(self, key, ndim, subset=None):

cython_cast_blacklist = frozenset(['rank', 'count', 'size', 'idxmin',
'idxmax'])


def whitelist_method_generator(base, klass, whitelist):
"""
Yields all GroupBy member defs for DataFrame/Series names in whitelist.

Parameters
----------
base : class
base class
klass : class
class where members are defined.
Should be Series or DataFrame
whitelist : list
list of names of klass methods to be constructed

Returns
-------
The generator yields a sequence of strings, each suitable for exec'ing,
that define implementations of the named methods for DataFrameGroupBy
or SeriesGroupBy.

Since we don't want to override methods explicitly defined in the
base class, any such name is skipped.
"""

method_wrapper_template = \
"""def %(name)s(%(sig)s) :
\"""
%(doc)s
\"""
f = %(self)s.__getattr__('%(name)s')
return f(%(args)s)"""
property_wrapper_template = \
"""@property
def %(name)s(self) :
\"""%(doc)s\"""
return self.__getattr__('%(name)s')"""

for name in whitelist:
# don't override anything that was explicitly defined
# in the base class
if hasattr(base, name):
continue
# ugly, but we need the name string itself in the method.
f = getattr(klass, name)
doc = f.__doc__
doc = doc if type(doc) == str else ''
if isinstance(f, types.MethodType):
wrapper_template = method_wrapper_template
decl, args = make_signature(f)
# pass args by name to f because otherwise
# GroupBy._make_wrapper won't know whether
# we passed in an axis parameter.
args_by_name = ['{0}={0}'.format(arg) for arg in args[1:]]
params = {'name': name,
'doc': doc,
'sig': ','.join(decl),
'self': args[0],
'args': ','.join(args_by_name)}
else:
wrapper_template = property_wrapper_template
params = {'name': name, 'doc': doc}
yield wrapper_template % params
52 changes: 49 additions & 3 deletions pandas/core/groupby/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from functools import partial
from textwrap import dedent
import typing
from typing import Any, Callable, List, Union
from typing import Any, Callable, FrozenSet, Iterator, List, Type, Union
import warnings

import numpy as np
Expand All @@ -27,6 +27,7 @@
is_integer_dtype, is_interval_dtype, is_numeric_dtype, is_scalar)
from pandas.core.dtypes.missing import isna, notna

from pandas._typing import FrameOrSeries
import pandas.core.algorithms as algorithms
from pandas.core.base import DataError, SpecificationError
import pandas.core.common as com
Expand All @@ -48,6 +49,51 @@
AggScalar = Union[str, Callable[..., Any]]


def whitelist_method_generator(base_class: Type[GroupBy],
klass: Type[FrameOrSeries],
whitelist: FrozenSet[str],
) -> Iterator[str]:
"""
Yields all GroupBy member defs for DataFrame/Series names in whitelist.

Parameters
----------
base_class : Groupby class
base class
klass : DataFrame or Series class
class where members are defined.
whitelist : frozenset
Set of names of klass methods to be constructed

Returns
-------
The generator yields a sequence of strings, each suitable for exec'ing,
that define implementations of the named methods for DataFrameGroupBy
or SeriesGroupBy.

Since we don't want to override methods explicitly defined in the
base class, any such name is skipped.
"""
property_wrapper_template = \
"""@property
def %(name)s(self) :
\"""%(doc)s\"""
return self.__getattr__('%(name)s')"""

for name in whitelist:
# don't override anything that was explicitly defined
# in the base class
if hasattr(base_class, name):
continue
# ugly, but we need the name string itself in the method.
f = getattr(klass, name)
doc = f.__doc__
doc = doc if type(doc) == str else ''
wrapper_template = property_wrapper_template
params = {'name': name, 'doc': doc}
yield wrapper_template % params


class NDFrameGroupBy(GroupBy):

def _iterate_slices(self):
Expand Down Expand Up @@ -685,7 +731,7 @@ class SeriesGroupBy(GroupBy):
# Make class defs of attributes on SeriesGroupBy whitelist

_apply_whitelist = base.series_apply_whitelist
for _def_str in base.whitelist_method_generator(
for _def_str in whitelist_method_generator(
GroupBy, Series, _apply_whitelist):
exec(_def_str)

Expand Down Expand Up @@ -1289,7 +1335,7 @@ class DataFrameGroupBy(NDFrameGroupBy):

#
# Make class defs of attributes on DataFrameGroupBy whitelist.
for _def_str in base.whitelist_method_generator(
for _def_str in whitelist_method_generator(
GroupBy, DataFrame, _apply_whitelist):
exec(_def_str)

Expand Down
18 changes: 0 additions & 18 deletions pandas/tests/util/test_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@

import pandas.compat as compat
from pandas.compat import raise_with_traceback
from pandas.util._decorators import deprecate_kwarg, make_signature
from pandas.util._validators import validate_kwargs

import pandas.util.testing as tm

Expand Down Expand Up @@ -37,22 +35,6 @@ def test_numpy_err_state_is_default():
assert np.geterr() == expected


@pytest.mark.parametrize("func,expected", [
# Case where the func does not have default kwargs.
(validate_kwargs, (["fname", "kwargs", "compat_args"],
["fname", "kwargs", "compat_args"])),

# Case where the func does have default kwargs.
(deprecate_kwarg, (["old_arg_name", "new_arg_name",
"mapping=None", "stacklevel=2"],
["old_arg_name", "new_arg_name",
"mapping", "stacklevel"]))
])
def test_make_signature(func, expected):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It can be noted that validate_kwargs and deprecate_kwarg have their own test modules, so removing here it not an issue.

# see gh-17608
assert make_signature(func) == expected


def test_raise_with_traceback():
with pytest.raises(LookupError, match="error_text"):
try:
Expand Down
30 changes: 0 additions & 30 deletions pandas/util/_decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,33 +319,3 @@ def indent(text, indents=1):
return ''
jointext = ''.join(['\n'] + [' '] * indents)
return jointext.join(text.split('\n'))


def make_signature(func):
"""
Returns a tuple containing the paramenter list with defaults
and parameter list.

Examples
--------
>>> def f(a, b, c=2):
>>> return a * b * c
>>> print(make_signature(f))
(['a', 'b', 'c=2'], ['a', 'b', 'c'])
"""

spec = inspect.getfullargspec(func)
if spec.defaults is None:
n_wo_defaults = len(spec.args)
defaults = ('',) * n_wo_defaults
else:
n_wo_defaults = len(spec.args) - len(spec.defaults)
defaults = ('',) * n_wo_defaults + tuple(spec.defaults)
args = []
for var, default in zip(spec.args, defaults):
args.append(var if default == '' else var + '=' + repr(default))
if spec.varargs:
args.append('*' + spec.varargs)
if spec.varkw:
args.append('**' + spec.varkw)
return args, spec.args