From 7b50b3efa6426e8f1107f962badbaec543c4a6b4 Mon Sep 17 00:00:00 2001 From: phofl Date: Wed, 2 Jun 2021 22:34:14 +0200 Subject: [PATCH 1/7] Bug in xs raising KeyError for MultiIndex columns with droplevel False and list indexer --- doc/source/whatsnew/v1.2.5.rst | 1 + pandas/core/generic.py | 2 +- pandas/tests/frame/indexing/test_xs.py | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.5.rst b/doc/source/whatsnew/v1.2.5.rst index 500030e1304c6..d3718ea0c6e1d 100644 --- a/doc/source/whatsnew/v1.2.5.rst +++ b/doc/source/whatsnew/v1.2.5.rst @@ -18,6 +18,7 @@ Fixed regressions - Fixed regression in :meth:`DataFrame.sum` and :meth:`DataFrame.prod` when ``min_count`` and ``numeric_only`` are both given (:issue:`41074`) - Regression in :func:`read_csv` when using ``memory_map=True`` with an non-UTF8 encoding (:issue:`40986`) - Regression in :meth:`DataFrame.replace` and :meth:`Series.replace` when the values to replace is a NumPy float array (:issue:`40371`) +- Regression in :meth:`DataFrame.xs` raising ``KeyError`` for existing element with list-like indexer for :class:`MultiIndex` columns with ``droplevel=False`` (:issue:`41760`) .. --------------------------------------------------------------------------- diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 49dc71954fd8f..4f7c32017ce5b 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3771,7 +3771,7 @@ class animal locomotion return result if axis == 1: - if drop_level: + if drop_level or not is_scalar(key): return self[key] index = self.columns else: diff --git a/pandas/tests/frame/indexing/test_xs.py b/pandas/tests/frame/indexing/test_xs.py index 57ea6a63f86e8..5188e519a2bcc 100644 --- a/pandas/tests/frame/indexing/test_xs.py +++ b/pandas/tests/frame/indexing/test_xs.py @@ -358,3 +358,11 @@ def test_xs_droplevel_false_view(self, using_array_manager): df.iloc[0, 0] = 2 expected = DataFrame({"a": [1]}) tm.assert_frame_equal(result, expected) + + def test_xs_list_indexer_droplevel_false(self): + # GH#41760 + mi = MultiIndex.from_tuples([("x", "y", "a"), ("x", "z", "b"), ("v", "w", "c")]) + df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=mi) + expected = df.copy() + result = df.xs(["x", "v"], drop_level=False, axis=1) + tm.assert_frame_equal(result, expected) From 6e23ba772dd049649a5d094e1bb2db6216de9a63 Mon Sep 17 00:00:00 2001 From: phofl Date: Thu, 3 Jun 2021 23:43:58 +0200 Subject: [PATCH 2/7] Deprecate list as key for same length as level and raise if lengths do not match --- doc/source/whatsnew/v1.2.5.rst | 1 - doc/source/whatsnew/v1.3.0.rst | 3 ++- pandas/core/generic.py | 14 +++++++++++++- pandas/tests/frame/indexing/test_xs.py | 15 ++++++++++----- pandas/tests/indexing/multiindex/test_partial.py | 2 +- pandas/tests/series/indexing/test_xs.py | 12 ++++++++++++ 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/doc/source/whatsnew/v1.2.5.rst b/doc/source/whatsnew/v1.2.5.rst index d3718ea0c6e1d..500030e1304c6 100644 --- a/doc/source/whatsnew/v1.2.5.rst +++ b/doc/source/whatsnew/v1.2.5.rst @@ -18,7 +18,6 @@ Fixed regressions - Fixed regression in :meth:`DataFrame.sum` and :meth:`DataFrame.prod` when ``min_count`` and ``numeric_only`` are both given (:issue:`41074`) - Regression in :func:`read_csv` when using ``memory_map=True`` with an non-UTF8 encoding (:issue:`40986`) - Regression in :meth:`DataFrame.replace` and :meth:`Series.replace` when the values to replace is a NumPy float array (:issue:`40371`) -- Regression in :meth:`DataFrame.xs` raising ``KeyError`` for existing element with list-like indexer for :class:`MultiIndex` columns with ``droplevel=False`` (:issue:`41760`) .. --------------------------------------------------------------------------- diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index b36499c340fd9..e2ec9ecd8c390 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -707,7 +707,7 @@ Deprecations - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_csv` (:issue:`41485`) - Deprecated passing arguments as positional in :meth:`DataFrame.drop` (other than ``"labels"``) and :meth:`Series.drop` (:issue:`41485`) - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_table` (:issue:`41485`) - +- Deprecated passing lists as ``key`` to :meth:`DataFrame.xs` and :meth:`Series.xs` with same length as ``level`` keyword (:issue:`41789`) .. _whatsnew_130.deprecations.nuisance_columns: @@ -945,6 +945,7 @@ Indexing - Bug in :meth:`DataFrame.loc` returning :class:`MultiIndex` in wrong order if indexer has duplicates (:issue:`40978`) - Bug in :meth:`DataFrame.__setitem__` raising ``TypeError`` when using a str subclass as the column name with a :class:`DatetimeIndex` (:issue:`37366`) - Bug in :meth:`PeriodIndex.get_loc` failing to raise ``KeyError`` when given a :class:`Period` with a mismatched ``freq`` (:issue:`41670`) +- Bug in :meth:`DataFrame.xs` and :meth:`Series.xs` accepting list as ``key`` and casting elements to different levels per index in list, now raises ``TypeError`` (:issue:`41760`) Missing ^^^^^^^ diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 4f7c32017ce5b..d9bc7853ef3ba 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3756,6 +3756,18 @@ class animal locomotion """ axis = self._get_axis_number(axis) labels = self._get_axis(axis) + + if isinstance(key, list): + len_levels = 1 if level is None or is_scalar(level) else len(level) + if len(key) != len_levels: + raise TypeError("Passing lists as key for xs is not allowed.") + warnings.warn( + "Passing lists as key for xs is deprecated and will be removed in a " + "future version. Pass key as a tuple instead.", + FutureWarning, + stacklevel=2, + ) + if level is not None: if not isinstance(labels, MultiIndex): raise TypeError("Index must be a MultiIndex") @@ -3771,7 +3783,7 @@ class animal locomotion return result if axis == 1: - if drop_level or not is_scalar(key): + if drop_level: return self[key] index = self.columns else: diff --git a/pandas/tests/frame/indexing/test_xs.py b/pandas/tests/frame/indexing/test_xs.py index 5188e519a2bcc..17c9365db5261 100644 --- a/pandas/tests/frame/indexing/test_xs.py +++ b/pandas/tests/frame/indexing/test_xs.py @@ -106,7 +106,8 @@ def test_xs_keep_level(self): expected = df[:1] tm.assert_frame_equal(result, expected) - result = df.xs([2008, "sat"], level=["year", "day"], drop_level=False) + with tm.assert_produces_warning(FutureWarning): + result = df.xs([2008, "sat"], level=["year", "day"], drop_level=False) tm.assert_frame_equal(result, expected) def test_xs_view(self, using_array_manager): @@ -187,7 +188,11 @@ def test_xs_with_duplicates(self, key, level, multiindex_dataframe_random_data): assert df.index.is_unique is False expected = concat([frame.xs("one", level="second")] * 2) - result = df.xs(key, level=level) + if isinstance(key, list): + with tm.assert_produces_warning(FutureWarning): + result = df.xs(key, level=level) + else: + result = df.xs(key, level=level) tm.assert_frame_equal(result, expected) def test_xs_missing_values_in_index(self): @@ -363,6 +368,6 @@ def test_xs_list_indexer_droplevel_false(self): # GH#41760 mi = MultiIndex.from_tuples([("x", "y", "a"), ("x", "z", "b"), ("v", "w", "c")]) df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=mi) - expected = df.copy() - result = df.xs(["x", "v"], drop_level=False, axis=1) - tm.assert_frame_equal(result, expected) + msg = "Passing lists as key for xs is not allowed." + with pytest.raises(TypeError, match=msg): + df.xs(["x", "v"], drop_level=False, axis=1) diff --git a/pandas/tests/indexing/multiindex/test_partial.py b/pandas/tests/indexing/multiindex/test_partial.py index 932295c28c8cf..1317ced185324 100644 --- a/pandas/tests/indexing/multiindex/test_partial.py +++ b/pandas/tests/indexing/multiindex/test_partial.py @@ -69,7 +69,7 @@ def test_xs_partial( ) df = DataFrame(np.random.randn(8, 4), index=index, columns=list("abcd")) - result = df.xs(["foo", "one"]) + result = df.xs(("foo", "one")) expected = df.loc["foo", "one"] tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/indexing/test_xs.py b/pandas/tests/series/indexing/test_xs.py index b6351e970222f..707d19ed6de8f 100644 --- a/pandas/tests/series/indexing/test_xs.py +++ b/pandas/tests/series/indexing/test_xs.py @@ -1,4 +1,5 @@ import numpy as np +import pytest from pandas import ( MultiIndex, @@ -69,3 +70,14 @@ def test_series_xs_droplevel_false(self): ), ) tm.assert_series_equal(result, expected) + + def test_xs_key_as_list(self): + # GH#41760 + mi = MultiIndex.from_tuples([("a", "x")], names=["level1", "level2"]) + ser = Series([1], index=mi) + msg = "Passing lists as key for xs is not allowed." + with pytest.raises(TypeError, match=msg): + ser.xs(["a", "x"], axis=0, drop_level=False) + + with tm.assert_produces_warning(FutureWarning): + ser.xs(["a"], axis=0, drop_level=False) From 986a9d910b470445928651c9ee0df2144be7909a Mon Sep 17 00:00:00 2001 From: phofl Date: Thu, 3 Jun 2021 23:55:10 +0200 Subject: [PATCH 3/7] Move whatsnew --- doc/source/whatsnew/v1.2.5.rst | 9 ++++++++- doc/source/whatsnew/v1.3.0.rst | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v1.2.5.rst b/doc/source/whatsnew/v1.2.5.rst index 500030e1304c6..778a459e2c5c3 100644 --- a/doc/source/whatsnew/v1.2.5.rst +++ b/doc/source/whatsnew/v1.2.5.rst @@ -21,12 +21,19 @@ Fixed regressions .. --------------------------------------------------------------------------- +.. _whatsnew_125.deprecations: + +Deprecations +~~~~~~~~~~~~ + +- Deprecated passing lists as ``key`` to :meth:`DataFrame.xs` and :meth:`Series.xs` with same length as ``level`` keyword (:issue:`41789`) + .. _whatsnew_125.bug_fixes: Bug fixes ~~~~~~~~~ -- +- Bug in :meth:`DataFrame.xs` and :meth:`Series.xs` accepting list as ``key`` and casting elements to different levels per index in list, now raises ``TypeError`` (:issue:`41760`) - .. --------------------------------------------------------------------------- diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index f9d1df7eb5786..022cd4ae79a6e 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -708,7 +708,6 @@ Deprecations - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_csv` (:issue:`41485`) - Deprecated passing arguments as positional in :meth:`DataFrame.drop` (other than ``"labels"``) and :meth:`Series.drop` (:issue:`41485`) - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_table` (:issue:`41485`) -- Deprecated passing lists as ``key`` to :meth:`DataFrame.xs` and :meth:`Series.xs` with same length as ``level`` keyword (:issue:`41789`) .. _whatsnew_130.deprecations.nuisance_columns: @@ -947,7 +946,6 @@ Indexing - Bug in :meth:`DataFrame.__setitem__` raising ``TypeError`` when using a str subclass as the column name with a :class:`DatetimeIndex` (:issue:`37366`) - Bug in :meth:`PeriodIndex.get_loc` failing to raise ``KeyError`` when given a :class:`Period` with a mismatched ``freq`` (:issue:`41670`) - Bug ``.loc.__getitem__`` with a :class:`UInt64Index` and negative-integer keys raising ``OverflowError`` instead of ``KeyError`` in some cases, wrapping around to positive integers in others (:issue:`41777`) -- Bug in :meth:`DataFrame.xs` and :meth:`Series.xs` accepting list as ``key`` and casting elements to different levels per index in list, now raises ``TypeError`` (:issue:`41760`) Missing ^^^^^^^ From 28ca6867e8ccfed19e0d191db4581c5f6ae0b8f8 Mon Sep 17 00:00:00 2001 From: phofl Date: Thu, 3 Jun 2021 23:57:46 +0200 Subject: [PATCH 4/7] Add minus back in --- doc/source/whatsnew/v1.3.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 022cd4ae79a6e..59abb73f2efef 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -709,6 +709,7 @@ Deprecations - Deprecated passing arguments as positional in :meth:`DataFrame.drop` (other than ``"labels"``) and :meth:`Series.drop` (:issue:`41485`) - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_table` (:issue:`41485`) +- .. _whatsnew_130.deprecations.nuisance_columns: Deprecated Dropping Nuisance Columns in DataFrame Reductions and DataFrameGroupBy Operations From bec4118b94f4f6594971f6e90d199ebc93719693 Mon Sep 17 00:00:00 2001 From: phofl Date: Thu, 3 Jun 2021 23:58:19 +0200 Subject: [PATCH 5/7] Add empty line --- doc/source/whatsnew/v1.3.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 59abb73f2efef..88c69335b39f4 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -709,7 +709,7 @@ Deprecations - Deprecated passing arguments as positional in :meth:`DataFrame.drop` (other than ``"labels"``) and :meth:`Series.drop` (:issue:`41485`) - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_table` (:issue:`41485`) -- + .. _whatsnew_130.deprecations.nuisance_columns: Deprecated Dropping Nuisance Columns in DataFrame Reductions and DataFrameGroupBy Operations From 3adcdd850f0527a8456989e6fe0684f114bcb1a2 Mon Sep 17 00:00:00 2001 From: phofl Date: Fri, 4 Jun 2021 00:05:50 +0200 Subject: [PATCH 6/7] Deprecate everything --- pandas/core/generic.py | 3 --- pandas/tests/frame/indexing/test_xs.py | 8 ++++---- pandas/tests/indexing/multiindex/test_partial.py | 3 ++- pandas/tests/series/indexing/test_xs.py | 4 +--- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 342b481780d8a..35e10b27e978c 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3758,9 +3758,6 @@ class animal locomotion labels = self._get_axis(axis) if isinstance(key, list): - len_levels = 1 if level is None or is_scalar(level) else len(level) - if len(key) != len_levels: - raise TypeError("Passing lists as key for xs is not allowed.") warnings.warn( "Passing lists as key for xs is deprecated and will be removed in a " "future version. Pass key as a tuple instead.", diff --git a/pandas/tests/frame/indexing/test_xs.py b/pandas/tests/frame/indexing/test_xs.py index 17c9365db5261..ccd989e2de411 100644 --- a/pandas/tests/frame/indexing/test_xs.py +++ b/pandas/tests/frame/indexing/test_xs.py @@ -366,8 +366,8 @@ def test_xs_droplevel_false_view(self, using_array_manager): def test_xs_list_indexer_droplevel_false(self): # GH#41760 - mi = MultiIndex.from_tuples([("x", "y", "a"), ("x", "z", "b"), ("v", "w", "c")]) + mi = MultiIndex.from_tuples([("x", "m", "a"), ("x", "n", "b"), ("y", "o", "c")]) df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=mi) - msg = "Passing lists as key for xs is not allowed." - with pytest.raises(TypeError, match=msg): - df.xs(["x", "v"], drop_level=False, axis=1) + with tm.assert_produces_warning(FutureWarning): + with pytest.raises(KeyError, match="y"): + df.xs(["x", "y"], drop_level=False, axis=1) diff --git a/pandas/tests/indexing/multiindex/test_partial.py b/pandas/tests/indexing/multiindex/test_partial.py index 1317ced185324..a99f09143e282 100644 --- a/pandas/tests/indexing/multiindex/test_partial.py +++ b/pandas/tests/indexing/multiindex/test_partial.py @@ -69,7 +69,8 @@ def test_xs_partial( ) df = DataFrame(np.random.randn(8, 4), index=index, columns=list("abcd")) - result = df.xs(("foo", "one")) + with tm.assert_produces_warning(FutureWarning): + result = df.xs(["foo", "one"]) expected = df.loc["foo", "one"] tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/indexing/test_xs.py b/pandas/tests/series/indexing/test_xs.py index 707d19ed6de8f..9a277783a1b3d 100644 --- a/pandas/tests/series/indexing/test_xs.py +++ b/pandas/tests/series/indexing/test_xs.py @@ -1,5 +1,4 @@ import numpy as np -import pytest from pandas import ( MultiIndex, @@ -75,8 +74,7 @@ def test_xs_key_as_list(self): # GH#41760 mi = MultiIndex.from_tuples([("a", "x")], names=["level1", "level2"]) ser = Series([1], index=mi) - msg = "Passing lists as key for xs is not allowed." - with pytest.raises(TypeError, match=msg): + with tm.assert_produces_warning(FutureWarning): ser.xs(["a", "x"], axis=0, drop_level=False) with tm.assert_produces_warning(FutureWarning): From e7fea60c4b64cbb6a6b94f361dd5e17653094213 Mon Sep 17 00:00:00 2001 From: phofl Date: Fri, 4 Jun 2021 00:06:41 +0200 Subject: [PATCH 7/7] Change whatsnew --- doc/source/whatsnew/v1.2.5.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v1.2.5.rst b/doc/source/whatsnew/v1.2.5.rst index 778a459e2c5c3..42c71acdee7e0 100644 --- a/doc/source/whatsnew/v1.2.5.rst +++ b/doc/source/whatsnew/v1.2.5.rst @@ -26,14 +26,14 @@ Fixed regressions Deprecations ~~~~~~~~~~~~ -- Deprecated passing lists as ``key`` to :meth:`DataFrame.xs` and :meth:`Series.xs` with same length as ``level`` keyword (:issue:`41789`) +- Deprecated passing lists as ``key`` to :meth:`DataFrame.xs` and :meth:`Series.xs` (:issue:`41760`) .. _whatsnew_125.bug_fixes: Bug fixes ~~~~~~~~~ -- Bug in :meth:`DataFrame.xs` and :meth:`Series.xs` accepting list as ``key`` and casting elements to different levels per index in list, now raises ``TypeError`` (:issue:`41760`) +- - .. ---------------------------------------------------------------------------