Skip to content

BUG/REF: Refactor BaseWindowIndex so groupby.apply has a consistent index output #39765

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Feb 25, 2021
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
861f762
Remove _groupby object from class
mroeschke Feb 10, 2021
8458946
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 11, 2021
0543909
Add test that groupby apply is not affected
mroeschke Feb 11, 2021
7e572be
Remove unnecessary groupby subclasses
mroeschke Feb 11, 2021
ef3f791
Supress private variables in repr
mroeschke Feb 11, 2021
3d76221
Don't have window classes accept kwargs
mroeschke Feb 11, 2021
97e772d
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 12, 2021
f914c66
Add whatsnew
mroeschke Feb 12, 2021
f330a78
Remove unused constructor
mroeschke Feb 12, 2021
c0b2474
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 15, 2021
e9dc172
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 16, 2021
adea173
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 19, 2021
9f23f54
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 22, 2021
194fe78
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 23, 2021
885745a
Just user the grouper
mroeschke Feb 23, 2021
54927a7
Make expected index instead
mroeschke Feb 23, 2021
6793dec
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 24, 2021
b4fdf86
Merge remote-tracking branch 'upstream/master' into cln/rollinggroupby
mroeschke Feb 24, 2021
33cfda7
Merge branch 'master' into cln/rollinggroupby
jreback Feb 25, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/source/whatsnew/v1.3.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ Groupby/resample/rolling
- Bug in :meth:`core.window.rolling.RollingGroupby.corr` and :meth:`core.window.expanding.ExpandingGroupby.corr` where the groupby column would return 0 instead of ``np.nan`` when providing ``other`` that was longer than each group (:issue:`39591`)
- Bug in :meth:`core.window.expanding.ExpandingGroupby.corr` and :meth:`core.window.expanding.ExpandingGroupby.cov` where 1 would be returned instead of ``np.nan`` when providing ``other`` that was longer than each group (:issue:`39591`)
- Bug in :meth:`.GroupBy.mean`, :meth:`.GroupBy.median` and :meth:`DataFrame.pivot_table` not propagating metadata (:issue:`28283`)
-
- Bug in :meth:`DataFrameGroupBy.apply` where a :class:`MultiIndex` would be created instead of an :class:`Index` if a :class:`:meth:`core.window.rolling.RollingGroupby` object was created (:issue:`39732`)

Reshaping
^^^^^^^^^
Expand Down
11 changes: 2 additions & 9 deletions pandas/core/groupby/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
"""
import collections
from typing import List
import warnings

from pandas._typing import final

Expand All @@ -31,10 +30,7 @@ def _shallow_copy(self, obj, **kwargs):
obj = obj.obj
for attr in self._attributes:
if attr not in kwargs:
# TODO: Remove once win_type deprecation is enforced
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "win_type", FutureWarning)
kwargs[attr] = getattr(self, attr)
kwargs[attr] = getattr(self, attr)
return self._constructor(obj, **kwargs)


Expand Down Expand Up @@ -65,10 +61,7 @@ def _gotitem(self, key, ndim, subset=None):

# we need to make a shallow copy of ourselves
# with the same groupby
# TODO: Remove once win_type deprecation is enforced
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "win_type", FutureWarning)
kwargs = {attr: getattr(self, attr) for attr in self._attributes}
kwargs = {attr: getattr(self, attr) for attr in self._attributes}

# Try to select from a DataFrame, falling back to a Series
try:
Expand Down
30 changes: 27 additions & 3 deletions pandas/core/groupby/groupby.py
Original file line number Diff line number Diff line change
Expand Up @@ -1918,7 +1918,15 @@ def rolling(self, *args, **kwargs):
"""
from pandas.core.window import RollingGroupby

return RollingGroupby(self, *args, **kwargs)
return RollingGroupby(
self._selected_obj,
*args,
_grouping_indices=self.grouper.indices,
_grouping_keys=self.grouper.names,
_grouping_codes=self.grouper.codes,
_grouping_levels=self.grouper.levels,
**kwargs,
)

@final
@Substitution(name="groupby")
Expand All @@ -1930,7 +1938,15 @@ def expanding(self, *args, **kwargs):
"""
from pandas.core.window import ExpandingGroupby

return ExpandingGroupby(self, *args, **kwargs)
return ExpandingGroupby(
self._selected_obj,
*args,
_grouping_indices=self.grouper.indices,
_grouping_keys=self.grouper.names,
_grouping_codes=self.grouper.codes,
_grouping_levels=self.grouper.levels,
**kwargs,
)

@final
@Substitution(name="groupby")
Expand All @@ -1941,7 +1957,15 @@ def ewm(self, *args, **kwargs):
"""
from pandas.core.window import ExponentialMovingWindowGroupby

return ExponentialMovingWindowGroupby(self, *args, **kwargs)
return ExponentialMovingWindowGroupby(
self._selected_obj,
*args,
_grouping_indices=self.grouper.indices,
_grouping_keys=self.grouper.names,
_grouping_codes=self.grouper.codes,
_grouping_levels=self.grouper.levels,
**kwargs,
)

@final
def _fill(self, direction, limit=None):
Expand Down
33 changes: 20 additions & 13 deletions pandas/core/window/ewm.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,15 @@ class ExponentialMovingWindow(BaseWindow):
4 3.233686
"""

_attributes = ["com", "min_periods", "adjust", "ignore_na", "axis"]
_attributes = [
"com",
"min_periods",
"adjust",
"ignore_na",
"axis",
"halflife",
"times",
]

def __init__(
self,
Expand All @@ -227,17 +235,18 @@ def __init__(
ignore_na: bool = False,
axis: int = 0,
times: Optional[Union[str, np.ndarray, FrameOrSeries]] = None,
**kwargs,
):
self.obj = obj
self.min_periods = max(int(min_periods), 1)
super().__init__(
obj=obj,
min_periods=max(int(min_periods), 1),
on=None,
center=False,
closed=None,
method="single",
axis=axis,
)
self.adjust = adjust
self.ignore_na = ignore_na
self.axis = axis
self.on = None
self.center = False
self.closed = None
self.method = "single"
if times is not None:
if isinstance(times, str):
times = self._selected_obj[times]
Expand Down Expand Up @@ -556,9 +565,7 @@ class ExponentialMovingWindowGroupby(BaseWindowGroupby, ExponentialMovingWindow)
Provide an exponential moving window groupby implementation.
"""

@property
def _constructor(self):
return ExponentialMovingWindow
_attributes = ExponentialMovingWindow._attributes + BaseWindowGroupby._attributes

def _get_window_indexer(self) -> GroupbyIndexer:
"""
Expand All @@ -569,7 +576,7 @@ def _get_window_indexer(self) -> GroupbyIndexer:
GroupbyIndexer
"""
window_indexer = GroupbyIndexer(
groupby_indicies=self._groupby.indices,
groupby_indicies=self._grouping_indices,
window_indexer=ExponentialMovingWindowIndexer,
)
return window_indexer
Expand Down
10 changes: 3 additions & 7 deletions pandas/core/window/expanding.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,7 @@ class Expanding(RollingAndExpandingMixin):

_attributes = ["min_periods", "center", "axis", "method"]

def __init__(
self, obj, min_periods=1, center=None, axis=0, method="single", **kwargs
):
def __init__(self, obj, min_periods=1, center=None, axis=0, method="single"):
super().__init__(
obj=obj, min_periods=min_periods, center=center, axis=axis, method=method
)
Expand Down Expand Up @@ -629,9 +627,7 @@ class ExpandingGroupby(BaseWindowGroupby, Expanding):
Provide a expanding groupby implementation.
"""

@property
def _constructor(self):
return Expanding
_attributes = Expanding._attributes + BaseWindowGroupby._attributes

def _get_window_indexer(self) -> GroupbyIndexer:
"""
Expand All @@ -642,7 +638,7 @@ def _get_window_indexer(self) -> GroupbyIndexer:
GroupbyIndexer
"""
window_indexer = GroupbyIndexer(
groupby_indicies=self._groupby.indices,
groupby_indicies=self._grouping_indices,
window_indexer=ExpandingIndexer,
)
return window_indexer
Loading