diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index 636bb2dc3e60e..0f67b58c678a7 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -418,6 +418,7 @@ Indexing - Bug in ``.isin()`` in which checking membership in empty ``Series`` objects raised an error (:issue:`16991`) - Bug in ``CategoricalIndex`` reindexing in which specified indices containing duplicates were not being respected (:issue:`17323`) - Bug in intersection of ``RangeIndex`` with negative step (:issue:`17296`) +- Bug in ``IntervalIndex`` where performing a scalar lookup fails for included right endpoints of non-overlapping monotonic decreasing indexes (:issue:`16417`, :issue:`17271`) I/O ^^^ diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index a9098126a38e3..ef5f68936044a 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -3465,7 +3465,7 @@ def _searchsorted_monotonic(self, label, side='left'): # everything for it to work (element ordering, search side and # resulting value). pos = self[::-1].searchsorted(label, side='right' if side == 'left' - else 'right') + else 'left') return len(self) - pos raise ValueError('index must be monotonic increasing or decreasing') diff --git a/pandas/tests/indexes/common.py b/pandas/tests/indexes/common.py index 1fdc08d68eb26..90618cd6e235f 100644 --- a/pandas/tests/indexes/common.py +++ b/pandas/tests/indexes/common.py @@ -11,6 +11,7 @@ RangeIndex, MultiIndex, CategoricalIndex, DatetimeIndex, TimedeltaIndex, PeriodIndex, IntervalIndex, notna, isna) +from pandas.core.indexes.base import InvalidIndexError from pandas.core.indexes.datetimelike import DatetimeIndexOpsMixin from pandas.core.dtypes.common import needs_i8_conversion from pandas._libs.tslib import iNaT @@ -138,9 +139,14 @@ def test_get_indexer_consistency(self): if isinstance(index, IntervalIndex): continue - indexer = index.get_indexer(index[0:2]) - assert isinstance(indexer, np.ndarray) - assert indexer.dtype == np.intp + if index.is_unique or isinstance(index, CategoricalIndex): + indexer = index.get_indexer(index[0:2]) + assert isinstance(indexer, np.ndarray) + assert indexer.dtype == np.intp + else: + e = "Reindexing only valid with uniquely valued Index objects" + with tm.assert_raises_regex(InvalidIndexError, e): + indexer = index.get_indexer(index[0:2]) indexer, _ = index.get_indexer_non_unique(index[0:2]) assert isinstance(indexer, np.ndarray) @@ -632,7 +638,8 @@ def test_difference_base(self): pass elif isinstance(idx, (DatetimeIndex, TimedeltaIndex)): assert result.__class__ == answer.__class__ - tm.assert_numpy_array_equal(result.asi8, answer.asi8) + tm.assert_numpy_array_equal(result.sort_values().asi8, + answer.sort_values().asi8) else: result = first.difference(case) assert tm.equalContents(result, answer) @@ -954,3 +961,47 @@ def test_join_self_unique(self, how): if index.is_unique: joined = index.join(index, how=how) assert (index == joined).all() + + def test_searchsorted_monotonic(self): + # GH17271 + for index in self.indices.values(): + # not implemented for tuple searches in MultiIndex + # or Intervals searches in IntervalIndex + if isinstance(index, (MultiIndex, IntervalIndex)): + continue + + # nothing to test if the index is empty + if index.empty: + continue + value = index[0] + + # determine the expected results (handle dupes for 'right') + expected_left, expected_right = 0, (index == value).argmin() + if expected_right == 0: + # all values are the same, expected_right should be length + expected_right = len(index) + + # test _searchsorted_monotonic in all cases + # test searchsorted only for increasing + if index.is_monotonic_increasing: + ssm_left = index._searchsorted_monotonic(value, side='left') + assert expected_left == ssm_left + + ssm_right = index._searchsorted_monotonic(value, side='right') + assert expected_right == ssm_right + + ss_left = index.searchsorted(value, side='left') + assert expected_left == ss_left + + ss_right = index.searchsorted(value, side='right') + assert expected_right == ss_right + elif index.is_monotonic_decreasing: + ssm_left = index._searchsorted_monotonic(value, side='left') + assert expected_left == ssm_left + + ssm_right = index._searchsorted_monotonic(value, side='right') + assert expected_right == ssm_right + else: + # non-monotonic should raise. + with pytest.raises(ValueError): + index._searchsorted_monotonic(value, side='left') diff --git a/pandas/tests/indexes/datetimes/test_datetimelike.py b/pandas/tests/indexes/datetimes/test_datetimelike.py index 3b970ee382521..538e10e6011ec 100644 --- a/pandas/tests/indexes/datetimes/test_datetimelike.py +++ b/pandas/tests/indexes/datetimes/test_datetimelike.py @@ -12,7 +12,9 @@ class TestDatetimeIndex(DatetimeLike): _holder = DatetimeIndex def setup_method(self, method): - self.indices = dict(index=tm.makeDateIndex(10)) + self.indices = dict(index=tm.makeDateIndex(10), + index_dec=date_range('20130110', periods=10, + freq='-1D')) self.setup_indices() def create_index(self): diff --git a/pandas/tests/indexes/period/test_period.py b/pandas/tests/indexes/period/test_period.py index e24e2ad936e2c..51f7d13cb0638 100644 --- a/pandas/tests/indexes/period/test_period.py +++ b/pandas/tests/indexes/period/test_period.py @@ -18,7 +18,9 @@ class TestPeriodIndex(DatetimeLike): _multiprocess_can_split_ = True def setup_method(self, method): - self.indices = dict(index=tm.makePeriodIndex(10)) + self.indices = dict(index=tm.makePeriodIndex(10), + index_dec=period_range('20130101', periods=10, + freq='D')[::-1]) self.setup_indices() def create_index(self): diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index f96dbdcfb8acf..d69fbbcdf4bf6 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -46,7 +46,8 @@ def setup_method(self, method): catIndex=tm.makeCategoricalIndex(100), empty=Index([]), tuples=MultiIndex.from_tuples(lzip( - ['foo', 'bar', 'baz'], [1, 2, 3]))) + ['foo', 'bar', 'baz'], [1, 2, 3])), + repeats=Index([0, 0, 1, 1, 2, 2])) self.setup_indices() def create_index(self): diff --git a/pandas/tests/indexes/test_numeric.py b/pandas/tests/indexes/test_numeric.py index 1a0a38c173284..7e7e10e4aeabe 100644 --- a/pandas/tests/indexes/test_numeric.py +++ b/pandas/tests/indexes/test_numeric.py @@ -181,7 +181,9 @@ class TestFloat64Index(Numeric): def setup_method(self, method): self.indices = dict(mixed=Float64Index([1.5, 2, 3, 4, 5]), - float=Float64Index(np.arange(5) * 2.5)) + float=Float64Index(np.arange(5) * 2.5), + mixed_dec=Float64Index([5, 4, 3, 2, 1.5]), + float_dec=Float64Index(np.arange(4, -1, -1) * 2.5)) self.setup_indices() def create_index(self): @@ -654,7 +656,8 @@ class TestInt64Index(NumericInt): _holder = Int64Index def setup_method(self, method): - self.indices = dict(index=Int64Index(np.arange(0, 20, 2))) + self.indices = dict(index=Int64Index(np.arange(0, 20, 2)), + index_dec=Int64Index(np.arange(19, -1, -1))) self.setup_indices() def create_index(self): @@ -949,8 +952,9 @@ class TestUInt64Index(NumericInt): _holder = UInt64Index def setup_method(self, method): - self.indices = dict(index=UInt64Index([2**63, 2**63 + 10, 2**63 + 15, - 2**63 + 20, 2**63 + 25])) + vals = [2**63, 2**63 + 10, 2**63 + 15, 2**63 + 20, 2**63 + 25] + self.indices = dict(index=UInt64Index(vals), + index_dec=UInt64Index(reversed(vals))) self.setup_indices() def create_index(self): diff --git a/pandas/tests/indexes/test_range.py b/pandas/tests/indexes/test_range.py index 06c8f0ee392c7..d206c36ee51c9 100644 --- a/pandas/tests/indexes/test_range.py +++ b/pandas/tests/indexes/test_range.py @@ -25,7 +25,8 @@ class TestRangeIndex(Numeric): _compat_props = ['shape', 'ndim', 'size', 'itemsize'] def setup_method(self, method): - self.indices = dict(index=RangeIndex(0, 20, 2, name='foo')) + self.indices = dict(index=RangeIndex(0, 20, 2, name='foo'), + index_dec=RangeIndex(18, -1, -2, name='bar')) self.setup_indices() def create_index(self): diff --git a/pandas/tests/indexing/test_interval.py b/pandas/tests/indexing/test_interval.py index be6e5e1cffb2e..31a94abcd99a5 100644 --- a/pandas/tests/indexing/test_interval.py +++ b/pandas/tests/indexing/test_interval.py @@ -3,6 +3,7 @@ import pandas as pd from pandas import Series, DataFrame, IntervalIndex, Interval +from pandas.compat import product import pandas.util.testing as tm @@ -14,16 +15,6 @@ def setup_method(self, method): def test_loc_with_scalar(self): s = self.s - expected = 0 - - result = s.loc[0.5] - assert result == expected - - result = s.loc[1] - assert result == expected - - with pytest.raises(KeyError): - s.loc[0] expected = s.iloc[:3] tm.assert_series_equal(expected, s.loc[:3]) @@ -42,16 +33,6 @@ def test_loc_with_scalar(self): def test_getitem_with_scalar(self): s = self.s - expected = 0 - - result = s[0.5] - assert result == expected - - result = s[1] - assert result == expected - - with pytest.raises(KeyError): - s[0] expected = s.iloc[:3] tm.assert_series_equal(expected, s[:3]) @@ -67,6 +48,41 @@ def test_getitem_with_scalar(self): expected = s.iloc[2:5] tm.assert_series_equal(expected, s[s >= 2]) + @pytest.mark.parametrize('direction, closed', + product(('increasing', 'decreasing'), + ('left', 'right', 'neither', 'both'))) + def test_nonoverlapping_monotonic(self, direction, closed): + tpls = [(0, 1), (2, 3), (4, 5)] + if direction == 'decreasing': + tpls = reversed(tpls) + + idx = IntervalIndex.from_tuples(tpls, closed=closed) + s = Series(list('abc'), idx) + + for key, expected in zip(idx.left, s): + if idx.closed_left: + assert s[key] == expected + assert s.loc[key] == expected + else: + with pytest.raises(KeyError): + s[key] + with pytest.raises(KeyError): + s.loc[key] + + for key, expected in zip(idx.right, s): + if idx.closed_right: + assert s[key] == expected + assert s.loc[key] == expected + else: + with pytest.raises(KeyError): + s[key] + with pytest.raises(KeyError): + s.loc[key] + + for key, expected in zip(idx.mid, s): + assert s[key] == expected + assert s.loc[key] == expected + def test_with_interval(self): s = self.s