From 1f23ebc92f6c15bf1c2dd7d7a1ddbfc5debd81a2 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Sun, 11 Nov 2018 15:01:13 -0600 Subject: [PATCH 1/6] BUG: Ensure that Index._data is an ndarray BUG: Ensure that Index._data is an ndarray Split from https://github.com/pandas-dev/pandas/pull/23623, where it was causing issues with infer_dtype. --- pandas/core/indexes/base.py | 4 ++++ pandas/tests/indexes/test_base.py | 6 ++++++ pandas/tests/indexes/test_numeric.py | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 263de57d32f31..e0a64897b3905 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -522,6 +522,10 @@ def _simple_new(cls, values, name=None, dtype=None, **kwargs): values = cls(values, name=name, dtype=dtype, **kwargs)._ndarray_values + if isinstance(values, (ABCSeries, cls)): + values = np.asarray(values._values) + + assert isinstance(values, np.ndarray) result = object.__new__(cls) result._data = values result.name = name diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 4a3efe22926f7..054efa00cd892 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -504,6 +504,12 @@ def test_constructor_cast(self): with pytest.raises(ValueError, match=msg): Index(["a", "b", "c"], dtype=float) + def test_constructor_unwraps_index(self): + a = pd.Index([True, False]) + b = pd.Index(a) + expected = np.array([True, False], dtype=object) + tm.assert_numpy_array_equal(b._data, expected) + def test_view_with_args(self): restricted = ['unicodeIndex', 'strIndex', 'catIndex', 'boolIndex', diff --git a/pandas/tests/indexes/test_numeric.py b/pandas/tests/indexes/test_numeric.py index c125db16bcbff..d1ad2308d19e5 100644 --- a/pandas/tests/indexes/test_numeric.py +++ b/pandas/tests/indexes/test_numeric.py @@ -628,6 +628,12 @@ def test_constructor_coercion_signed_to_unsigned(self, uint_dtype): with pytest.raises(OverflowError, match=msg): Index([-1], dtype=uint_dtype) + def test_constructor_unwraps_index(self): + idx = pd.Index([1, 2]) + result = pd.Int64Index(idx) + expected = np.array([1, 2], dtype='int64') + tm.assert_numpy_array_equal(result._data, expected) + def test_coerce_list(self): # coerce things arr = Index([1, 2, 3, 4]) From a30bc02e51afc8b8c152561ec11c9f0190736aec Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Sun, 11 Nov 2018 15:14:46 -0600 Subject: [PATCH 2/6] remove assert --- pandas/core/indexes/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index e0a64897b3905..43939e415c583 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -525,7 +525,6 @@ def _simple_new(cls, values, name=None, dtype=None, **kwargs): if isinstance(values, (ABCSeries, cls)): values = np.asarray(values._values) - assert isinstance(values, np.ndarray) result = object.__new__(cls) result._data = values result.name = name From fa8934a3082300c209b4dddc9eb7e34029b6493f Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 12 Nov 2018 06:16:53 -0600 Subject: [PATCH 3/6] update errors --- pandas/tests/series/test_operators.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 4cce26d135443..653ef6fa7920d 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -14,6 +14,7 @@ from pandas import ( Categorical, DataFrame, Index, NaT, Series, bdate_range, date_range, isna) from pandas.core import ops +from pandas.core.indexes.base import InvalidIndexError import pandas.core.nanops as nanops import pandas.util.testing as tm from pandas.util.testing import ( @@ -198,11 +199,14 @@ def test_scalar_na_logical_ops_corners(self): pytest.param(ops.ror_, marks=pytest.mark.xfail(reason="GH#22092 Index " "implementation raises", - raises=ValueError, strict=True)), + raises=InvalidIndexError, + strict=True)), pytest.param(ops.rxor, marks=pytest.mark.xfail(reason="GH#22092 Index " - "implementation raises", - raises=TypeError, strict=True)) + "implementation returns " + "Index", + raises=AssertionError, + strict=True)) ]) def test_logical_ops_with_index(self, op): # GH#22092, GH#19792 From e9035505281e7ec73e5dd0a372f30124b4c6327b Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 12 Nov 2018 09:31:38 -0600 Subject: [PATCH 4/6] Add note [ci skip] ***NO CI*** --- pandas/core/indexes/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 43939e415c583..0099149286e14 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -523,6 +523,9 @@ def _simple_new(cls, values, name=None, dtype=None, **kwargs): **kwargs)._ndarray_values if isinstance(values, (ABCSeries, cls)): + # Index._data must always be an ndarray. + # This is no-copy for when _values is an ndarray, + # which should be always at this point. values = np.asarray(values._values) result = object.__new__(cls) From e4b21f60c504b33f179007412cb385517f977e28 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 12 Nov 2018 16:09:58 -0600 Subject: [PATCH 5/6] TST: Change rops tests --- pandas/tests/series/test_operators.py | 32 ++++++++++++--------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 653ef6fa7920d..bcecedc2bba97 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -14,7 +14,6 @@ from pandas import ( Categorical, DataFrame, Index, NaT, Series, bdate_range, date_range, isna) from pandas.core import ops -from pandas.core.indexes.base import InvalidIndexError import pandas.core.nanops as nanops import pandas.util.testing as tm from pandas.util.testing import ( @@ -190,23 +189,7 @@ def test_scalar_na_logical_ops_corners(self): operator.and_, operator.or_, operator.xor, - pytest.param(ops.rand_, - marks=pytest.mark.xfail(reason="GH#22092 Index " - "implementation returns " - "Index", - raises=AssertionError, - strict=True)), - pytest.param(ops.ror_, - marks=pytest.mark.xfail(reason="GH#22092 Index " - "implementation raises", - raises=InvalidIndexError, - strict=True)), - pytest.param(ops.rxor, - marks=pytest.mark.xfail(reason="GH#22092 Index " - "implementation returns " - "Index", - raises=AssertionError, - strict=True)) + ]) def test_logical_ops_with_index(self, op): # GH#22092, GH#19792 @@ -225,6 +208,19 @@ def test_logical_ops_with_index(self, op): result = op(ser, idx2) assert_series_equal(result, expected) + @pytest.mark.parametrize("op, expected", [ + (ops.rand_, pd.Index([False, True])), + (ops.ror_, pd.Index([False, True])), + (ops.rxor, pd.Index([])), + ]) + def test_reverse_ops_with_index(self, op, expected): + # https://github.com/pandas-dev/pandas/pull/23628 + # multi-set Index ops are buggy, so let's avoid duplicates... + ser = Series([True, False]) + idx = Index([False, True]) + result = op(ser, idx) + tm.assert_index_equal(result, expected) + def test_logical_ops_label_based(self): # GH#4947 # logical ops should be label based From 265ffcb70dce16599d1464d31458adb23c9666a7 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 14 Nov 2018 15:14:32 -0600 Subject: [PATCH 6/6] parametrize --- pandas/core/indexes/base.py | 4 ++-- pandas/tests/indexes/test_base.py | 11 ++++++----- pandas/tests/indexes/test_numeric.py | 6 ------ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 8c4edf1f05c34..454d029c266f5 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -19,7 +19,7 @@ ABCSeries, ABCDataFrame, ABCMultiIndex, ABCPeriodIndex, ABCTimedeltaIndex, ABCDatetimeIndex, - ABCDateOffset) + ABCDateOffset, ABCIndexClass) from pandas.core.dtypes.missing import isna, array_equivalent from pandas.core.dtypes.cast import maybe_cast_to_integer_array from pandas.core.dtypes.common import ( @@ -522,7 +522,7 @@ def _simple_new(cls, values, name=None, dtype=None, **kwargs): values = cls(values, name=name, dtype=dtype, **kwargs)._ndarray_values - if isinstance(values, (ABCSeries, cls)): + if isinstance(values, (ABCSeries, ABCIndexClass)): # Index._data must always be an ndarray. # This is no-copy for when _values is an ndarray, # which should be always at this point. diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index e836f43d18d4f..424f6b1f9a77a 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -504,11 +504,12 @@ def test_constructor_cast(self): with pytest.raises(ValueError, match=msg): Index(["a", "b", "c"], dtype=float) - def test_constructor_unwraps_index(self): - a = pd.Index([True, False]) - b = pd.Index(a) - expected = np.array([True, False], dtype=object) - tm.assert_numpy_array_equal(b._data, expected) + def test_constructor_unwraps_index(self, indices): + if isinstance(indices, pd.MultiIndex): + raise pytest.skip("MultiIndex has no ._data") + a = indices + b = type(a)(a) + tm.assert_equal(a._data, b._data) def test_view_with_args(self): diff --git a/pandas/tests/indexes/test_numeric.py b/pandas/tests/indexes/test_numeric.py index d1ad2308d19e5..c125db16bcbff 100644 --- a/pandas/tests/indexes/test_numeric.py +++ b/pandas/tests/indexes/test_numeric.py @@ -628,12 +628,6 @@ def test_constructor_coercion_signed_to_unsigned(self, uint_dtype): with pytest.raises(OverflowError, match=msg): Index([-1], dtype=uint_dtype) - def test_constructor_unwraps_index(self): - idx = pd.Index([1, 2]) - result = pd.Int64Index(idx) - expected = np.array([1, 2], dtype='int64') - tm.assert_numpy_array_equal(result._data, expected) - def test_coerce_list(self): # coerce things arr = Index([1, 2, 3, 4])