From c47d36f7e0ff032f52eff9375895d5d8a4e1c476 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 6 Mar 2019 11:09:38 +0000 Subject: [PATCH 1/7] DEPR: remove Panel-specific parts of core.indexing --- pandas/core/indexing.py | 88 +++-------------------------------------- 1 file changed, 6 insertions(+), 82 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 623a48acdd48b..11c1876af970a 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -14,7 +14,7 @@ from pandas.core.dtypes.common import ( ensure_platform_int, is_float, is_integer, is_integer_dtype, is_iterator, is_list_like, is_scalar, is_sequence, is_sparse) -from pandas.core.dtypes.generic import ABCDataFrame, ABCPanel, ABCSeries +from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries from pandas.core.dtypes.missing import _infer_fill_value, isna import pandas.core.common as com @@ -453,10 +453,6 @@ def _setitem_with_indexer(self, indexer, value): self.obj._maybe_update_cacher(clear=True) return self.obj - # set using setitem (Panel and > dims) - elif self.ndim >= 3: - return self.obj.__setitem__(indexer, value) - # set item_labels = self.obj._get_axis(info_axis) @@ -645,9 +641,6 @@ def can_do_equal_len(): elif isinstance(value, ABCDataFrame): value = self._align_frame(indexer, value) - if isinstance(value, ABCPanel): - value = self._align_panel(indexer, value) - # check for chained assignment self.obj._check_is_chained_assignment_possible() @@ -693,7 +686,6 @@ def ravel(i): sum_aligners = sum(aligners) single_aligner = sum_aligners == 1 is_frame = self.obj.ndim == 2 - is_panel = self.obj.ndim >= 3 obj = self.obj # are we a single alignable value on a non-primary @@ -705,11 +697,6 @@ def ravel(i): if is_frame: single_aligner = single_aligner and aligners[0] - # panel - elif is_panel: - single_aligner = (single_aligner and - (aligners[1] or aligners[2])) - # we have a frame, with multiple indexers on both axes; and a # series, so need to broadcast (see GH5206) if (sum_aligners == self.ndim and @@ -741,7 +728,7 @@ def ravel(i): return ser.reindex(new_ix)._values # 2 dims - elif single_aligner and is_frame: + elif single_aligner: # reindex along index ax = self.obj.axes[1] @@ -749,30 +736,6 @@ def ravel(i): return ser._values.copy() return ser.reindex(ax)._values - # >2 dims - elif single_aligner: - - broadcast = [] - for n, labels in enumerate(self.obj._get_plane_axes(i)): - - # reindex along the matching dimensions - if len(labels & ser.index): - ser = ser.reindex(labels) - else: - broadcast.append((n, len(labels))) - - # broadcast along other dims - ser = ser._values.copy() - for (axis, l) in broadcast: - shape = [-1] * (len(broadcast) + 1) - shape[axis] = l - ser = np.tile(ser, l).reshape(shape) - - if self.obj.ndim == 3: - ser = ser.T - - return ser - elif is_scalar(indexer): ax = self.obj._get_axis(1) @@ -785,7 +748,6 @@ def ravel(i): def _align_frame(self, indexer, df): is_frame = self.obj.ndim == 2 - is_panel = self.obj.ndim >= 3 if isinstance(indexer, tuple): @@ -805,21 +767,6 @@ def _align_frame(self, indexer, df): else: sindexers.append(i) - # panel - if is_panel: - - # need to conform to the convention - # as we are not selecting on the items axis - # and we have a single indexer - # GH 7763 - if len(sindexers) == 1 and sindexers[0] != 0: - df = df.T - - if idx is None: - idx = df.index - if cols is None: - cols = df.columns - if idx is not None and cols is not None: if df.index.equals(idx) and df.columns.equals(cols): @@ -846,24 +793,8 @@ def _align_frame(self, indexer, df): val = df.reindex(index=ax)._values return val - elif is_scalar(indexer) and is_panel: - idx = self.obj.axes[1] - cols = self.obj.axes[2] - - # by definition we are indexing on the 0th axis - # a passed in dataframe which is actually a transpose - # of what is needed - if idx.equals(df.index) and cols.equals(df.columns): - return df.copy()._values - - return df.reindex(idx, columns=cols)._values - raise ValueError('Incompatible indexer with DataFrame') - def _align_panel(self, indexer, df): - raise NotImplementedError("cannot set using an indexer with a Panel " - "yet!") - def _getitem_tuple(self, tup): try: return self._getitem_lowerdim(tup) @@ -1056,13 +987,6 @@ def _getitem_nested_tuple(self, tup): # has the dim of the obj changed? # GH 7199 if obj.ndim < current_ndim: - - # GH 7516 - # if had a 3 dim and are going to a 2d - # axes are reversed on a DataFrame - if i >= 1 and current_ndim == 3 and obj.ndim == 2: - obj = obj.T - axis -= 1 return obj @@ -1559,8 +1483,8 @@ class _LocIndexer(_LocationIndexer): - A boolean array of the same length as the axis being sliced, e.g. ``[True, False, True]``. - - A ``callable`` function with one argument (the calling Series, DataFrame - or Panel) and that returns valid output for indexing (one of the above) + - A ``callable`` function with one argument (the calling Series or + DataFrame) and that returns valid output for indexing (one of the above) See more at :ref:`Selection by Label ` @@ -1929,8 +1853,8 @@ class _iLocIndexer(_LocationIndexer): - A list or array of integers, e.g. ``[4, 3, 0]``. - A slice object with ints, e.g. ``1:7``. - A boolean array. - - A ``callable`` function with one argument (the calling Series, DataFrame - or Panel) and that returns valid output for indexing (one of the above). + - A ``callable`` function with one argument (the calling Series or + DataFrame) and that returns valid output for indexing (one of the above). This is useful in method chains, when you don't have a reference to the calling object, but would like to base your selection on some value. From 348c595c78383dcba75871de924bd1f8116eef01 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Fri, 8 Mar 2019 14:03:06 +0000 Subject: [PATCH 2/7] add new error if ndim > 2 and remove failing test --- pandas/_libs/indexing.pyx | 1 + pandas/tests/generic/test_generic.py | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/indexing.pyx b/pandas/_libs/indexing.pyx index af6e00bad7f6b..f67783238f137 100644 --- a/pandas/_libs/indexing.pyx +++ b/pandas/_libs/indexing.pyx @@ -20,4 +20,5 @@ cdef class _NDFrameIndexerBase: ndim = self._ndim if ndim is None: ndim = self._ndim = self.obj.ndim + assert not ndim > 2, 'Internal Error: NDFrameIndexer ndim > 2' return ndim diff --git a/pandas/tests/generic/test_generic.py b/pandas/tests/generic/test_generic.py index c2f6cbf4c564c..c6c3c24a28f17 100644 --- a/pandas/tests/generic/test_generic.py +++ b/pandas/tests/generic/test_generic.py @@ -740,12 +740,9 @@ def test_squeeze(self): # don't fail with 0 length dimensions GH11229 & GH8999 empty_series = Series([], name='five') empty_frame = DataFrame([empty_series]) - with catch_warnings(record=True): - simplefilter("ignore", FutureWarning) - empty_panel = Panel({'six': empty_frame}) [tm.assert_series_equal(empty_series, higher_dim.squeeze()) - for higher_dim in [empty_series, empty_frame, empty_panel]] + for higher_dim in [empty_series, empty_frame]] # axis argument df = tm.makeTimeDataFrame(nper=1).iloc[:, :1] From 451a96776c8f592e27fe9e8cc1ae125d2baec217 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Fri, 8 Mar 2019 15:51:36 +0000 Subject: [PATCH 3/7] minor change --- pandas/_libs/indexing.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/indexing.pyx b/pandas/_libs/indexing.pyx index f67783238f137..49acc7a7c6143 100644 --- a/pandas/_libs/indexing.pyx +++ b/pandas/_libs/indexing.pyx @@ -20,5 +20,5 @@ cdef class _NDFrameIndexerBase: ndim = self._ndim if ndim is None: ndim = self._ndim = self.obj.ndim - assert not ndim > 2, 'Internal Error: NDFrameIndexer ndim > 2' + assert not ndim > 2, 'Internal Error: NDFrameIndexer ndim > 2' return ndim From f39a75fc252c3c967bc2a212e383870b5dadf2a7 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Fri, 22 Mar 2019 11:39:33 +0000 Subject: [PATCH 4/7] add 3-d indexer setting and getting tests --- pandas/tests/indexing/test_indexing.py | 89 ++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 03f1975c50d2a..2a155a11821bd 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -21,6 +21,8 @@ from pandas.tests.indexing.common import Base, _mklbl import pandas.util.testing as tm +ignore_ix = pytest.mark.filterwarnings("ignore:\\n.ix:DeprecationWarning") + # ------------------------------------------------------------------------ # Indexing test cases @@ -58,6 +60,93 @@ def test_setitem_ndarray_1d(self): with pytest.raises(ValueError): df[2:5] = np.arange(1, 4) * 1j + @pytest.mark.parametrize('index', tm.all_index_generator(5), + ids=lambda x: type(x).__name__) + @pytest.mark.parametrize('obj', [ + lambda i: Series(np.arange(len(i)), index=i), + lambda i: DataFrame( + np.random.randn(len(i), len(i)), index=i, columns=i) + ], ids=['Series', 'DataFrame']) + @pytest.mark.parametrize('idxr, idxr_id', [ + (lambda x: x, 'getitem'), + (lambda x: x.loc, 'loc'), + (lambda x: x.iloc, 'iloc'), + pytest.param(lambda x: x.ix, 'ix', marks=ignore_ix) + ]) + def test_getitem_ndarray_3d(self, index, obj, idxr, idxr_id): + # GH 25567 + obj = obj(index) + idxr = idxr(obj) + nd3 = np.random.randint(5, size=(2, 2, 2)) + + msg = (r"Buffer has wrong number of dimensions \(expected 1," + r" got 3\)|" + "The truth value of an array with more than one element is" + " ambiguous|" + "Cannot index with multidimensional key|" + r"Wrong number of dimensions. values.ndim != ndim \[3 != 1\]|" + "unhashable type: 'numpy.ndarray'" # TypeError + ) + + if (isinstance(obj, Series) and idxr_id == 'getitem' + and index.inferred_type in [ + 'string', 'datetime64', 'period', 'timedelta64', + 'boolean', 'categorical']): + idxr[nd3] + else: + if (isinstance(obj, DataFrame) and idxr_id == 'getitem' + and index.inferred_type == 'boolean'): + error = TypeError + else: + error = ValueError + + with pytest.raises(error, match=msg): + idxr[nd3] + + @pytest.mark.parametrize('index', tm.all_index_generator(5), + ids=lambda x: type(x).__name__) + @pytest.mark.parametrize('obj', [ + lambda i: Series(np.arange(len(i)), index=i), + lambda i: DataFrame( + np.random.randn(len(i), len(i)), index=i, columns=i) + ], ids=['Series', 'DataFrame']) + @pytest.mark.parametrize('idxr, idxr_id', [ + (lambda x: x, 'setitem'), + (lambda x: x.loc, 'loc'), + (lambda x: x.iloc, 'iloc'), + pytest.param(lambda x: x.ix, 'ix', marks=ignore_ix) + ]) + def test_setitem_ndarray_3d(self, index, obj, idxr, idxr_id): + # GH 25567 + obj = obj(index) + idxr = idxr(obj) + nd3 = np.random.randint(5, size=(2, 2, 2)) + + msg = (r"Buffer has wrong number of dimensions \(expected 1," + r" got 3\)|" + "The truth value of an array with more than one element is" + " ambiguous|" + "Only 1-dimensional input arrays are supported|" + "'pandas._libs.interval.IntervalTree' object has no attribute" + " 'set_value'|" # AttributeError + "unhashable type: 'numpy.ndarray'|" # TypeError + r"^\[\[\[" # pandas.core.indexing.IndexingError + ) + + if ((idxr_id == 'iloc') + or ((isinstance(obj, Series) and idxr_id == 'setitem' + and index.inferred_type in [ + 'floating', 'string', 'datetime64', 'period', 'timedelta64', + 'boolean', 'categorical'])) + or (idxr_id == 'ix' and index.inferred_type in [ + 'string', 'datetime64', 'period', 'boolean'])): + idxr[nd3] = 0 + else: + with pytest.raises( + (ValueError, AttributeError, TypeError, + pd.core.indexing.IndexingError), match=msg): + idxr[nd3] = 0 + def test_inf_upcast(self): # GH 16957 # We should be able to use np.inf as a key From 26f2ac21248c28afc07a6135601cbbfcdf806aca Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 30 Mar 2019 14:38:59 +0000 Subject: [PATCH 5/7] change exception raised and add test --- pandas/_libs/indexing.pyx | 5 ++++- pandas/tests/indexing/test_indexing.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/indexing.pyx b/pandas/_libs/indexing.pyx index 49acc7a7c6143..2fdca80d56d41 100644 --- a/pandas/_libs/indexing.pyx +++ b/pandas/_libs/indexing.pyx @@ -20,5 +20,8 @@ cdef class _NDFrameIndexerBase: ndim = self._ndim if ndim is None: ndim = self._ndim = self.obj.ndim - assert not ndim > 2, 'Internal Error: NDFrameIndexer ndim > 2' + if ndim > 2: + msg = ("NDFrameIndexer does not support NDFrame objects with" + " ndim > 2") + raise ValueError(msg) return ndim diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index fae4b616c20c8..f28d8a9a72c95 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -16,6 +16,7 @@ import pandas as pd from pandas import DataFrame, Index, NaT, Series +from pandas.core.generic import NDFrame from pandas.core.indexing import ( _maybe_numeric_slice, _non_reducing_slice, validate_indices) from pandas.tests.indexing.common import Base, _mklbl @@ -1100,3 +1101,16 @@ def test_extension_array_cross_section_converts(): result = df.iloc[0] tm.assert_series_equal(result, expected) + + +@pytest.mark.parametrize('idxr, idxr_id', [ + (lambda x: x, 'getitem'), + (lambda x: x.loc, 'loc'), + (lambda x: x.iloc, 'iloc'), + pytest.param(lambda x: x.ix, 'ix', marks=ignore_ix) +]) +def test_ndframe_indexing_raises(idxr, idxr_id): + frame = NDFrame(np.random.randint(5, size=(2, 2, 2))) + msg = "NDFrameIndexer does not support NDFrame objects with ndim > 2" + with pytest.raises(ValueError, match=msg): + idxr(frame)[0] From 1de10b51623fc4108ad932e5a2bdec70d21c5008 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Mon, 1 Apr 2019 07:01:17 +0100 Subject: [PATCH 6/7] fix linting errors --- pandas/_libs/indexing.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/indexing.pyx b/pandas/_libs/indexing.pyx index 2fdca80d56d41..0ec4fd3a37713 100644 --- a/pandas/_libs/indexing.pyx +++ b/pandas/_libs/indexing.pyx @@ -20,8 +20,8 @@ cdef class _NDFrameIndexerBase: ndim = self._ndim if ndim is None: ndim = self._ndim = self.obj.ndim - if ndim > 2: + if ndim > 2: msg = ("NDFrameIndexer does not support NDFrame objects with" - " ndim > 2") + " ndim > 2") raise ValueError(msg) return ndim From eaf161b362a0b452f41847908c73787c7768113c Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Thu, 30 May 2019 13:23:11 +0100 Subject: [PATCH 7/7] fix-up test_ndframe_indexing_raises --- pandas/tests/indexing/test_indexing.py | 28 +++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 9a39355f014cb..a0e3df182b129 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -17,7 +17,7 @@ from pandas.tests.indexing.common import Base, _mklbl import pandas.util.testing as tm -ignore_ix = pytest.mark.filterwarnings("ignore:\\n.ix:DeprecationWarning") +ignore_ix = pytest.mark.filterwarnings("ignore:\\n.ix:FutureWarning") # ------------------------------------------------------------------------ # Indexing test cases @@ -1107,14 +1107,24 @@ def test_extension_array_cross_section_converts(): tm.assert_series_equal(result, expected) -@pytest.mark.parametrize('idxr, idxr_id', [ - (lambda x: x, 'getitem'), - (lambda x: x.loc, 'loc'), - (lambda x: x.iloc, 'iloc'), - pytest.param(lambda x: x.ix, 'ix', marks=ignore_ix) +@pytest.mark.parametrize('idxr, error, error_message', [ + (lambda x: x, + AttributeError, + "'numpy.ndarray' object has no attribute 'get'"), + (lambda x: x.loc, + AttributeError, + "type object 'NDFrame' has no attribute '_AXIS_ALIASES'"), + (lambda x: x.iloc, + AttributeError, + "type object 'NDFrame' has no attribute '_AXIS_ALIASES'"), + pytest.param( + lambda x: x.ix, + ValueError, + "NDFrameIndexer does not support NDFrame objects with ndim > 2", + marks=ignore_ix) ]) -def test_ndframe_indexing_raises(idxr, idxr_id): +def test_ndframe_indexing_raises(idxr, error, error_message): + # GH 25567 frame = NDFrame(np.random.randint(5, size=(2, 2, 2))) - msg = "NDFrameIndexer does not support NDFrame objects with ndim > 2" - with pytest.raises(ValueError, match=msg): + with pytest.raises(error, match=error_message): idxr(frame)[0]