From 4b232a2b67ffdc662b8a2b77bee375e9aac612ff Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 27 Jan 2020 14:02:34 -0600 Subject: [PATCH 1/7] Avoid Index DeprecationWarning in Series getitem xref https://github.com/pandas-dev/pandas/issues/30867 --- pandas/core/indexes/base.py | 10 +++++++++- pandas/core/internals/managers.py | 6 +++++- pandas/tests/series/indexing/test_indexing.py | 9 +++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 4573fd7fc6da3..1a11654ab0fd5 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -4168,6 +4168,13 @@ def __setitem__(self, key, value): raise TypeError("Index does not support mutable operations") def __getitem__(self, key): + # To support the deprecation of Index[:, None] returning an + # ndarray with a warning, we use a helper method _getitem_deprecate_nd. + # Series[:, None] eventaully calls that with warn=False, via + # SingleBlockManager.get_slice + return self._getitem_deprecate_nd(key) + + def _getitem_deprecate_nd(self, key, warn=True): """ Override numpy.ndarray's __getitem__ method to work as desired. @@ -4199,7 +4206,8 @@ def __getitem__(self, key): result = getitem(key) if not is_scalar(result): if np.ndim(result) > 1: - deprecate_ndim_indexing(result) + if warn: + deprecate_ndim_indexing(result) return result return promote(result) else: diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index a3675a1a58fd3..35b77830c7797 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -1505,7 +1505,11 @@ def get_slice(self, slobj, axis=0): if axis >= self.ndim: raise IndexError("Requested axis not found in manager") - return type(self)(self._block._slice(slobj), self.index[slobj], fastpath=True) + return type(self)( + self._block._slice(slobj), + self.index._getitem_deprecate_nd(slobj, warn=False), + fastpath=True, + ) @property def index(self): diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 65731cf45bd2d..1717c369a2919 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -925,3 +925,12 @@ def test_uint_drop(any_int_dtype): series.loc[0] = 4 expected = pd.Series([4, 2, 3], dtype=any_int_dtype) tm.assert_series_equal(series, expected) + + +def test_getitem_2d_no_warning(): + # https://github.com/pandas-dev/pandas/issues/30867 + # Unclear that we want to support this long-term, but + # for now ensure that no warning from Index comes through. + series = pd.Series([1, 2, 3], index=[1, 2, 3]) + with tm.assert_produces_warning(None): + series[:, None] From 3a430b3a96fc157f4ebcd50a0359960399392a06 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 27 Jan 2020 14:08:30 -0600 Subject: [PATCH 2/7] clarify --- pandas/tests/series/indexing/test_indexing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 1717c369a2919..18dbd22b73b35 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -929,8 +929,9 @@ def test_uint_drop(any_int_dtype): def test_getitem_2d_no_warning(): # https://github.com/pandas-dev/pandas/issues/30867 - # Unclear that we want to support this long-term, but - # for now ensure that no warning from Index comes through. + # Don't want to support this long-term, but + # for now ensure that the warning from Index + # doesn't comes through via Series.__getitem__. series = pd.Series([1, 2, 3], index=[1, 2, 3]) with tm.assert_produces_warning(None): series[:, None] From 6a3acb29c662972cf58ec4a220d5e934d5c7ac27 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 27 Jan 2020 15:44:40 -0600 Subject: [PATCH 3/7] filter --- pandas/core/indexes/base.py | 12 +++--------- pandas/core/internals/managers.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 1a11654ab0fd5..f1e1d07925bb6 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -4168,13 +4168,6 @@ def __setitem__(self, key, value): raise TypeError("Index does not support mutable operations") def __getitem__(self, key): - # To support the deprecation of Index[:, None] returning an - # ndarray with a warning, we use a helper method _getitem_deprecate_nd. - # Series[:, None] eventaully calls that with warn=False, via - # SingleBlockManager.get_slice - return self._getitem_deprecate_nd(key) - - def _getitem_deprecate_nd(self, key, warn=True): """ Override numpy.ndarray's __getitem__ method to work as desired. @@ -4206,8 +4199,7 @@ def _getitem_deprecate_nd(self, key, warn=True): result = getitem(key) if not is_scalar(result): if np.ndim(result) > 1: - if warn: - deprecate_ndim_indexing(result) + deprecate_ndim_indexing(result) return result return promote(result) else: @@ -5834,6 +5826,8 @@ def deprecate_ndim_indexing(result): # GH#27125 indexer like idx[:, None] expands dim, but we # cannot do that and keep an index, so return ndarray # Deprecation GH#30588 + # Note: update SingleBlockManager.get_slice when the DeprecationWarning + # is elevated to a FutureWarning warnings.warn( "Support for multi-dimensional indexing (e.g. `index[:, None]`) " "on an Index is deprecated and will be removed in a future " diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 35b77830c7797..95d702c12f16a 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -4,6 +4,7 @@ import operator import re from typing import Dict, List, Optional, Sequence, Tuple, Union +import warnings import numpy as np @@ -1505,11 +1506,13 @@ def get_slice(self, slobj, axis=0): if axis >= self.ndim: raise IndexError("Requested axis not found in manager") - return type(self)( - self._block._slice(slobj), - self.index._getitem_deprecate_nd(slobj, warn=False), - fastpath=True, - ) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", "Support for multi-dim", DeprecationWarning + ) + index = self.index[slobj] + + return type(self)(self._block._slice(slobj), index, fastpath=True,) @property def index(self): From fc14a3588bd352714798eff3e3d2e4a49abd8f06 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 28 Jan 2020 08:14:04 -0600 Subject: [PATCH 4/7] perf --- pandas/core/internals/managers.py | 9 +-------- pandas/core/series.py | 6 +++++- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 95d702c12f16a..526863d2e5ec3 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -4,7 +4,6 @@ import operator import re from typing import Dict, List, Optional, Sequence, Tuple, Union -import warnings import numpy as np @@ -1506,13 +1505,7 @@ def get_slice(self, slobj, axis=0): if axis >= self.ndim: raise IndexError("Requested axis not found in manager") - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "Support for multi-dim", DeprecationWarning - ) - index = self.index[slobj] - - return type(self)(self._block._slice(slobj), index, fastpath=True,) + return type(self)(self._block._slice(slobj), self.index[slobj], fastpath=True,) @property def index(self): diff --git a/pandas/core/series.py b/pandas/core/series.py index e9df0938d5f98..dc16c1794c7dd 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -937,7 +937,11 @@ def _get_with(self, key): def _get_values_tuple(self, key): # mpl hackaround if com.any_none(*key): - return self._get_values(key) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", "Support for multi-dim", DeprecationWarning + ) + return self._get_values(key) if not isinstance(self.index, MultiIndex): raise ValueError("Can only tuple-index with a MultiIndex") From bd17044e5893354d684ac4c6c87b136873eedc17 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 28 Jan 2020 08:17:59 -0600 Subject: [PATCH 5/7] comment --- pandas/core/series.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/series.py b/pandas/core/series.py index dc16c1794c7dd..f5171f57de38f 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -937,6 +937,8 @@ def _get_with(self, key): def _get_values_tuple(self, key): # mpl hackaround if com.any_none(*key): + # suppress warning from slicing the index with a 2d indexer. + # eventually we'll want Series itself to warn. with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "Support for multi-dim", DeprecationWarning From 90b1e604eef1b420254c3b65d6f3e0804f1b1211 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 28 Jan 2020 08:54:39 -0600 Subject: [PATCH 6/7] comment --- pandas/tests/series/test_timeseries.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/tests/series/test_timeseries.py b/pandas/tests/series/test_timeseries.py index a2d14f27d7b7a..319b6d5bb6c2b 100644 --- a/pandas/tests/series/test_timeseries.py +++ b/pandas/tests/series/test_timeseries.py @@ -136,7 +136,11 @@ def test_first_last_valid(self, datetime_series): assert ts.first_valid_index().freq == ts.index.freq assert ts.last_valid_index().freq == ts.index.freq + @pytest.mark.xfail(reason="Deprecate in Series.__getitem__") def test_mpl_compat_hack(self, datetime_series): + # This is currently failing because the test was relying on + # the DeprecationWarning coming through Index.__getitem__. + # We want to implement a warning specifically for Series.__getitem__. with tm.assert_produces_warning(DeprecationWarning, check_stacklevel=False): # GH#30588 multi-dimensional indexing deprecated result = datetime_series[:, np.newaxis] From 476251343f66b783058e5e711636a1bfa72c6908 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 28 Jan 2020 09:37:02 -0600 Subject: [PATCH 7/7] unxfail --- pandas/tests/series/test_timeseries.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/tests/series/test_timeseries.py b/pandas/tests/series/test_timeseries.py index 319b6d5bb6c2b..459377fb18f29 100644 --- a/pandas/tests/series/test_timeseries.py +++ b/pandas/tests/series/test_timeseries.py @@ -136,12 +136,13 @@ def test_first_last_valid(self, datetime_series): assert ts.first_valid_index().freq == ts.index.freq assert ts.last_valid_index().freq == ts.index.freq - @pytest.mark.xfail(reason="Deprecate in Series.__getitem__") def test_mpl_compat_hack(self, datetime_series): + # This is currently failing because the test was relying on # the DeprecationWarning coming through Index.__getitem__. - # We want to implement a warning specifically for Series.__getitem__. - with tm.assert_produces_warning(DeprecationWarning, check_stacklevel=False): + # We want to implement a warning specifically for Series.__getitem__ + # at which point this will become a Deprecation/FutureWarning + with tm.assert_produces_warning(None): # GH#30588 multi-dimensional indexing deprecated result = datetime_series[:, np.newaxis] expected = datetime_series.values[:, np.newaxis]