From 287dd8289b51adb7acaee5282bca917241550abe Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 6 Sep 2020 01:14:22 +0200 Subject: [PATCH 1/4] Fix compressed multiindex for output of groupby.rolling --- doc/source/whatsnew/v1.2.0.rst | 1 + pandas/core/window/rolling.py | 10 ++++++---- pandas/tests/groupby/test_groupby.py | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index b1229a5d5823d..39305fefbfc31 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -314,6 +314,7 @@ Groupby/resample/rolling - Bug when subsetting columns on a :class:`~pandas.core.groupby.DataFrameGroupBy` (e.g. ``df.groupby('a')[['b']])``) would reset the attributes ``axis``, ``dropna``, ``group_keys``, ``level``, ``mutated``, ``sort``, and ``squeeze`` to their default values. (:issue:`9959`) - Bug in :meth:`DataFrameGroupby.tshift` failing to raise ``ValueError`` when a frequency cannot be inferred for the index of a group (:issue:`35937`) - Bug in :meth:`DataFrame.groupby` does not always maintain column index name for ``any``, ``all``, ``bfill``, ``ffill``, ``shift`` (:issue:`29764`) +- Bug in :meth:`Series.groupby.rolling` compressed number of levels of :class:`MultiIndex` in input to one (:issue:`36018`) - Reshaping diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 558c0eeb0ea65..8b85ff2c45eae 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -2211,17 +2211,19 @@ def _apply( # Compose MultiIndex result from grouping levels then rolling level # Aggregate the MultiIndex data as tuples then the level names grouped_object_index = self.obj.index - grouped_index_name = [grouped_object_index.name] + grouped_index_name = [*grouped_object_index.names] groupby_keys = [grouping.name for grouping in self._groupby.grouper._groupings] result_index_names = groupby_keys + grouped_index_name result_index_data = [] for key, values in self._groupby.grouper.indices.items(): for value in values: - if not is_list_like(key): - data = [key, grouped_object_index[value]] + data = [key] if not is_list_like(key) else [*key] + if is_list_like(grouped_object_index[value]): + grouped_object_index_value = [*grouped_object_index[value]] else: - data = [*key, grouped_object_index[value]] + grouped_object_index_value = [grouped_object_index[value]] + data.extend(grouped_object_index_value) result_index_data.append(tuple(data)) result_index = MultiIndex.from_tuples( diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index e0196df7ceac0..010dcc39f5ab4 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -2134,3 +2134,24 @@ def test_groupby_column_index_name_lost_fill_funcs(func): result = getattr(df_grouped, func)().columns expected = pd.Index(["a", "b"], name="idx") tm.assert_index_equal(result, expected) + + +@pytest.mark.parametrize("func", ["max", "min"]) +def test_groupby_rolling_index_changed(func): + ds = pd.Series( + [1, 2, 2], + index=pd.MultiIndex.from_tuples( + [("a", "x"), ("a", "y"), ("c", "z")], names=["1", "2"] + ), + name="a", + ) + + result = getattr(ds.groupby(ds).rolling(2), func)() + expected = pd.Series( + [np.nan, np.nan, 2.0], + index=pd.MultiIndex.from_tuples( + [(1, "a", "x"), (2, "a", "y"), (2, "c", "z")], names=["a", "1", "2"] + ), + name="a", + ) + tm.assert_series_equal(result, expected) From 2d4461c23a38d9c98925f83d7aadf0f2f952c661 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 6 Sep 2020 13:14:17 +0200 Subject: [PATCH 2/4] Add comment to test --- pandas/tests/groupby/test_groupby.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 010dcc39f5ab4..da0cb0c160902 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -2138,6 +2138,7 @@ def test_groupby_column_index_name_lost_fill_funcs(func): @pytest.mark.parametrize("func", ["max", "min"]) def test_groupby_rolling_index_changed(func): + # GH: #36018 nlevels of MultiIndex changed ds = pd.Series( [1, 2, 2], index=pd.MultiIndex.from_tuples( From e29ed53b68ddd7e5bc0090869bbec97b199c2ec6 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 6 Sep 2020 15:59:56 +0200 Subject: [PATCH 3/4] Move tests --- pandas/tests/groupby/test_groupby.py | 22 ---------------------- pandas/tests/window/test_grouper.py | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index da0cb0c160902..e0196df7ceac0 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -2134,25 +2134,3 @@ def test_groupby_column_index_name_lost_fill_funcs(func): result = getattr(df_grouped, func)().columns expected = pd.Index(["a", "b"], name="idx") tm.assert_index_equal(result, expected) - - -@pytest.mark.parametrize("func", ["max", "min"]) -def test_groupby_rolling_index_changed(func): - # GH: #36018 nlevels of MultiIndex changed - ds = pd.Series( - [1, 2, 2], - index=pd.MultiIndex.from_tuples( - [("a", "x"), ("a", "y"), ("c", "z")], names=["1", "2"] - ), - name="a", - ) - - result = getattr(ds.groupby(ds).rolling(2), func)() - expected = pd.Series( - [np.nan, np.nan, 2.0], - index=pd.MultiIndex.from_tuples( - [(1, "a", "x"), (2, "a", "y"), (2, "c", "z")], names=["a", "1", "2"] - ), - name="a", - ) - tm.assert_series_equal(result, expected) diff --git a/pandas/tests/window/test_grouper.py b/pandas/tests/window/test_grouper.py index 170bf100b3891..cb85ad7584da7 100644 --- a/pandas/tests/window/test_grouper.py +++ b/pandas/tests/window/test_grouper.py @@ -372,3 +372,24 @@ def test_groupby_subset_rolling_subset_with_closed(self): name="column1", ) tm.assert_series_equal(result, expected) + + @pytest.mark.parametrize("func", ["max", "min"]) + def test_groupby_rolling_index_changed(self, func): + # GH: #36018 nlevels of MultiIndex changed + ds = Series( + [1, 2, 2], + index=pd.MultiIndex.from_tuples( + [("a", "x"), ("a", "y"), ("c", "z")], names=["1", "2"] + ), + name="a", + ) + + result = getattr(ds.groupby(ds).rolling(2), func)() + expected = Series( + [np.nan, np.nan, 2.0], + index=pd.MultiIndex.from_tuples( + [(1, "a", "x"), (2, "a", "y"), (2, "c", "z")], names=["a", "1", "2"] + ), + name="a", + ) + tm.assert_series_equal(result, expected) From 025144021c7a3b5b6452cc524e45f3f7eca63725 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 6 Sep 2020 21:17:27 +0200 Subject: [PATCH 4/4] Reformat code and move whats new --- doc/source/whatsnew/v1.1.2.rst | 1 + doc/source/whatsnew/v1.2.0.rst | 1 - pandas/core/window/rolling.py | 10 ++++------ 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/source/whatsnew/v1.1.2.rst b/doc/source/whatsnew/v1.1.2.rst index d1a66256454ca..6a21f36284996 100644 --- a/doc/source/whatsnew/v1.1.2.rst +++ b/doc/source/whatsnew/v1.1.2.rst @@ -21,6 +21,7 @@ Fixed regressions - Regression in :meth:`DataFrame.replace` where a ``TypeError`` would be raised when attempting to replace elements of type :class:`Interval` (:issue:`35931`) - Fix regression in pickle roundtrip of the ``closed`` attribute of :class:`IntervalIndex` (:issue:`35658`) - Fixed regression in :meth:`DataFrameGroupBy.agg` where a ``ValueError: buffer source array is read-only`` would be raised when the underlying array is read-only (:issue:`36014`) +- Fixed regression in :meth:`Series.groupby.rolling` number of levels of :class:`MultiIndex` in input was compressed to one (:issue:`36018`) - .. --------------------------------------------------------------------------- diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 39305fefbfc31..b1229a5d5823d 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -314,7 +314,6 @@ Groupby/resample/rolling - Bug when subsetting columns on a :class:`~pandas.core.groupby.DataFrameGroupBy` (e.g. ``df.groupby('a')[['b']])``) would reset the attributes ``axis``, ``dropna``, ``group_keys``, ``level``, ``mutated``, ``sort``, and ``squeeze`` to their default values. (:issue:`9959`) - Bug in :meth:`DataFrameGroupby.tshift` failing to raise ``ValueError`` when a frequency cannot be inferred for the index of a group (:issue:`35937`) - Bug in :meth:`DataFrame.groupby` does not always maintain column index name for ``any``, ``all``, ``bfill``, ``ffill``, ``shift`` (:issue:`29764`) -- Bug in :meth:`Series.groupby.rolling` compressed number of levels of :class:`MultiIndex` in input to one (:issue:`36018`) - Reshaping diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 8b85ff2c45eae..875de28439ed4 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -2218,12 +2218,10 @@ def _apply( result_index_data = [] for key, values in self._groupby.grouper.indices.items(): for value in values: - data = [key] if not is_list_like(key) else [*key] - if is_list_like(grouped_object_index[value]): - grouped_object_index_value = [*grouped_object_index[value]] - else: - grouped_object_index_value = [grouped_object_index[value]] - data.extend(grouped_object_index_value) + data = [ + *com.maybe_make_list(key), + *com.maybe_make_list(grouped_object_index[value]), + ] result_index_data.append(tuple(data)) result_index = MultiIndex.from_tuples(