From fd6869993747072c9f921c46875c9f6ad41058d5 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Fri, 24 Apr 2020 12:05:33 -0500 Subject: [PATCH 1/5] BUG: Adjust truncate for decreasing index --- doc/source/whatsnew/v1.1.0.rst | 1 + pandas/core/generic.py | 3 +++ pandas/tests/frame/methods/test_truncate.py | 12 ++++++++++++ pandas/tests/series/methods/test_truncate.py | 12 ++++++++++++ 4 files changed, 28 insertions(+) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index cd1cb0b64f74a..95aede6d14d6c 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -531,6 +531,7 @@ Indexing - Bug in :meth:`DatetimeIndex.insert` and :meth:`TimedeltaIndex.insert` causing index ``freq`` to be lost when setting an element into an empty :class:`Series` (:issue:33573`) - Bug in :meth:`Series.__setitem__` with an :class:`IntervalIndex` and a list-like key of integers (:issue:`33473`) - Bug in :meth:`Series.__getitem__` allowing missing labels with ``np.ndarray``, :class:`Index`, :class:`Series` indexers but not ``list``, these now all raise ``KeyError`` (:issue:`33646`) +- Bug in :meth:`DataFrame.truncate` and :meth:`Series.truncate` where index was assumed to be monotone increasing (:issue:`33756`) Missing ^^^^^^^ diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 495dcc5700241..05d7ad05166e5 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -9195,6 +9195,9 @@ def truncate( if before > after: raise ValueError(f"Truncate: {after} must be after {before}") + if ax.is_monotonic_decreasing: + before, after = after, before + slicer = [slice(None, None)] * self._AXIS_LEN slicer[axis] = slice(before, after) result = self.loc[tuple(slicer)] diff --git a/pandas/tests/frame/methods/test_truncate.py b/pandas/tests/frame/methods/test_truncate.py index ad86ee1266874..7ab49f3e1c03a 100644 --- a/pandas/tests/frame/methods/test_truncate.py +++ b/pandas/tests/frame/methods/test_truncate.py @@ -87,3 +87,15 @@ def test_truncate_nonsortedindex(self): msg = "truncate requires a sorted index" with pytest.raises(ValueError, match=msg): df.truncate(before=2, after=20, axis=1) + + @pytest.mark.parametrize( + "before, after, indices", + [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], + ) + def test_truncate_decreasing_index(self, before, after, indices): + # https://github.com/pandas-dev/pandas/issues/33756 + idx = pd.Index([3, 2, 1, 0]) + values = pd.DataFrame(index=idx) + result = values.truncate(before=before, after=after) + expected = values.loc[indices] + tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/methods/test_truncate.py b/pandas/tests/series/methods/test_truncate.py index c97369b349f56..46e71410403c7 100644 --- a/pandas/tests/series/methods/test_truncate.py +++ b/pandas/tests/series/methods/test_truncate.py @@ -80,6 +80,18 @@ def test_truncate_nonsortedindex(self): with pytest.raises(ValueError, match=msg): ts.sort_values(ascending=False).truncate(before="2011-11", after="2011-12") + @pytest.mark.parametrize( + "before, after, indices", + [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], + ) + def test_truncate_decreasing_index(self, before, after, indices): + # https://github.com/pandas-dev/pandas/issues/33756 + idx = pd.Index([3, 2, 1, 0]) + values = pd.Series(index=idx) + result = values.truncate(before=before, after=after) + expected = values.loc[indices] + tm.assert_series_equal(result, expected) + def test_truncate_datetimeindex_tz(self): # GH 9243 idx = date_range("4/1/2005", "4/30/2005", freq="D", tz="US/Pacific") From 12955dca930bb6c34228adcfbd0595db792b9a9b Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Fri, 24 Apr 2020 12:48:24 -0500 Subject: [PATCH 2/5] Try with data --- pandas/tests/frame/methods/test_truncate.py | 2 +- pandas/tests/series/methods/test_truncate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/frame/methods/test_truncate.py b/pandas/tests/frame/methods/test_truncate.py index 7ab49f3e1c03a..875b0b056088a 100644 --- a/pandas/tests/frame/methods/test_truncate.py +++ b/pandas/tests/frame/methods/test_truncate.py @@ -95,7 +95,7 @@ def test_truncate_nonsortedindex(self): def test_truncate_decreasing_index(self, before, after, indices): # https://github.com/pandas-dev/pandas/issues/33756 idx = pd.Index([3, 2, 1, 0]) - values = pd.DataFrame(index=idx) + values = pd.DataFrame(range(len(idx)), index=idx) result = values.truncate(before=before, after=after) expected = values.loc[indices] tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/methods/test_truncate.py b/pandas/tests/series/methods/test_truncate.py index 46e71410403c7..aad2bb50f0e17 100644 --- a/pandas/tests/series/methods/test_truncate.py +++ b/pandas/tests/series/methods/test_truncate.py @@ -87,7 +87,7 @@ def test_truncate_nonsortedindex(self): def test_truncate_decreasing_index(self, before, after, indices): # https://github.com/pandas-dev/pandas/issues/33756 idx = pd.Index([3, 2, 1, 0]) - values = pd.Series(index=idx) + values = pd.Series(range(len(idx)), index=idx) result = values.truncate(before=before, after=after) expected = values.loc[indices] tm.assert_series_equal(result, expected) From 15e919d65dda683a4c89941518492cd8f5c50a8e Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Fri, 24 Apr 2020 18:06:24 -0500 Subject: [PATCH 3/5] Move tests --- pandas/tests/frame/methods/test_truncate.py | 12 ----------- pandas/tests/generic/methods/test_truncate.py | 21 +++++++++++++++++++ pandas/tests/series/methods/test_truncate.py | 12 ----------- 3 files changed, 21 insertions(+), 24 deletions(-) create mode 100644 pandas/tests/generic/methods/test_truncate.py diff --git a/pandas/tests/frame/methods/test_truncate.py b/pandas/tests/frame/methods/test_truncate.py index 875b0b056088a..ad86ee1266874 100644 --- a/pandas/tests/frame/methods/test_truncate.py +++ b/pandas/tests/frame/methods/test_truncate.py @@ -87,15 +87,3 @@ def test_truncate_nonsortedindex(self): msg = "truncate requires a sorted index" with pytest.raises(ValueError, match=msg): df.truncate(before=2, after=20, axis=1) - - @pytest.mark.parametrize( - "before, after, indices", - [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], - ) - def test_truncate_decreasing_index(self, before, after, indices): - # https://github.com/pandas-dev/pandas/issues/33756 - idx = pd.Index([3, 2, 1, 0]) - values = pd.DataFrame(range(len(idx)), index=idx) - result = values.truncate(before=before, after=after) - expected = values.loc[indices] - tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/generic/methods/test_truncate.py b/pandas/tests/generic/methods/test_truncate.py new file mode 100644 index 0000000000000..54cd44b9240a1 --- /dev/null +++ b/pandas/tests/generic/methods/test_truncate.py @@ -0,0 +1,21 @@ +import pytest + +from pandas import DataFrame, Index, Series +import pandas._testing as tm + + +@pytest.mark.parametrize( + "before, after, indices", + [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], +) +@pytest.mark.parametrize("klass", [Series, DataFrame]) +def test_truncate_decreasing_index(before, after, indices, klass): + # https://github.com/pandas-dev/pandas/issues/33756 + idx = Index([3, 2, 1, 0]) + values = klass(range(len(idx)), index=idx) + result = values.truncate(before=before, after=after) + expected = values.loc[indices] + if klass is Series: + tm.assert_series_equal(result, expected) + else: + tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/methods/test_truncate.py b/pandas/tests/series/methods/test_truncate.py index aad2bb50f0e17..c97369b349f56 100644 --- a/pandas/tests/series/methods/test_truncate.py +++ b/pandas/tests/series/methods/test_truncate.py @@ -80,18 +80,6 @@ def test_truncate_nonsortedindex(self): with pytest.raises(ValueError, match=msg): ts.sort_values(ascending=False).truncate(before="2011-11", after="2011-12") - @pytest.mark.parametrize( - "before, after, indices", - [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], - ) - def test_truncate_decreasing_index(self, before, after, indices): - # https://github.com/pandas-dev/pandas/issues/33756 - idx = pd.Index([3, 2, 1, 0]) - values = pd.Series(range(len(idx)), index=idx) - result = values.truncate(before=before, after=after) - expected = values.loc[indices] - tm.assert_series_equal(result, expected) - def test_truncate_datetimeindex_tz(self): # GH 9243 idx = date_range("4/1/2005", "4/30/2005", freq="D", tz="US/Pacific") From 3f06c9beddc023e3b5c920011e76ea6de4ed9ab5 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Fri, 24 Apr 2020 19:31:01 -0500 Subject: [PATCH 4/5] Revert "Move tests" This reverts commit 15e919d65dda683a4c89941518492cd8f5c50a8e. --- pandas/tests/frame/methods/test_truncate.py | 12 +++++++++++ pandas/tests/generic/methods/test_truncate.py | 21 ------------------- pandas/tests/series/methods/test_truncate.py | 12 +++++++++++ 3 files changed, 24 insertions(+), 21 deletions(-) delete mode 100644 pandas/tests/generic/methods/test_truncate.py diff --git a/pandas/tests/frame/methods/test_truncate.py b/pandas/tests/frame/methods/test_truncate.py index ad86ee1266874..875b0b056088a 100644 --- a/pandas/tests/frame/methods/test_truncate.py +++ b/pandas/tests/frame/methods/test_truncate.py @@ -87,3 +87,15 @@ def test_truncate_nonsortedindex(self): msg = "truncate requires a sorted index" with pytest.raises(ValueError, match=msg): df.truncate(before=2, after=20, axis=1) + + @pytest.mark.parametrize( + "before, after, indices", + [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], + ) + def test_truncate_decreasing_index(self, before, after, indices): + # https://github.com/pandas-dev/pandas/issues/33756 + idx = pd.Index([3, 2, 1, 0]) + values = pd.DataFrame(range(len(idx)), index=idx) + result = values.truncate(before=before, after=after) + expected = values.loc[indices] + tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/generic/methods/test_truncate.py b/pandas/tests/generic/methods/test_truncate.py deleted file mode 100644 index 54cd44b9240a1..0000000000000 --- a/pandas/tests/generic/methods/test_truncate.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from pandas import DataFrame, Index, Series -import pandas._testing as tm - - -@pytest.mark.parametrize( - "before, after, indices", - [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], -) -@pytest.mark.parametrize("klass", [Series, DataFrame]) -def test_truncate_decreasing_index(before, after, indices, klass): - # https://github.com/pandas-dev/pandas/issues/33756 - idx = Index([3, 2, 1, 0]) - values = klass(range(len(idx)), index=idx) - result = values.truncate(before=before, after=after) - expected = values.loc[indices] - if klass is Series: - tm.assert_series_equal(result, expected) - else: - tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/series/methods/test_truncate.py b/pandas/tests/series/methods/test_truncate.py index c97369b349f56..aad2bb50f0e17 100644 --- a/pandas/tests/series/methods/test_truncate.py +++ b/pandas/tests/series/methods/test_truncate.py @@ -80,6 +80,18 @@ def test_truncate_nonsortedindex(self): with pytest.raises(ValueError, match=msg): ts.sort_values(ascending=False).truncate(before="2011-11", after="2011-12") + @pytest.mark.parametrize( + "before, after, indices", + [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], + ) + def test_truncate_decreasing_index(self, before, after, indices): + # https://github.com/pandas-dev/pandas/issues/33756 + idx = pd.Index([3, 2, 1, 0]) + values = pd.Series(range(len(idx)), index=idx) + result = values.truncate(before=before, after=after) + expected = values.loc[indices] + tm.assert_series_equal(result, expected) + def test_truncate_datetimeindex_tz(self): # GH 9243 idx = date_range("4/1/2005", "4/30/2005", freq="D", tz="US/Pacific") From 85fcb0ca93eeae6d7ba573092572a08efb994a90 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Sat, 25 Apr 2020 18:41:05 -0500 Subject: [PATCH 5/5] Test DatetimeIndex --- pandas/tests/frame/methods/test_truncate.py | 9 +++++++-- pandas/tests/series/methods/test_truncate.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pandas/tests/frame/methods/test_truncate.py b/pandas/tests/frame/methods/test_truncate.py index 875b0b056088a..768a5f22fb063 100644 --- a/pandas/tests/frame/methods/test_truncate.py +++ b/pandas/tests/frame/methods/test_truncate.py @@ -92,9 +92,14 @@ def test_truncate_nonsortedindex(self): "before, after, indices", [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], ) - def test_truncate_decreasing_index(self, before, after, indices): + @pytest.mark.parametrize("klass", [pd.Int64Index, pd.DatetimeIndex]) + def test_truncate_decreasing_index(self, before, after, indices, klass): # https://github.com/pandas-dev/pandas/issues/33756 - idx = pd.Index([3, 2, 1, 0]) + idx = klass([3, 2, 1, 0]) + if klass is pd.DatetimeIndex: + before = pd.Timestamp(before) if before is not None else None + after = pd.Timestamp(after) if after is not None else None + indices = [pd.Timestamp(i) for i in indices] values = pd.DataFrame(range(len(idx)), index=idx) result = values.truncate(before=before, after=after) expected = values.loc[indices] diff --git a/pandas/tests/series/methods/test_truncate.py b/pandas/tests/series/methods/test_truncate.py index aad2bb50f0e17..47947f0287494 100644 --- a/pandas/tests/series/methods/test_truncate.py +++ b/pandas/tests/series/methods/test_truncate.py @@ -84,9 +84,14 @@ def test_truncate_nonsortedindex(self): "before, after, indices", [(1, 2, [2, 1]), (None, 2, [2, 1, 0]), (1, None, [3, 2, 1])], ) - def test_truncate_decreasing_index(self, before, after, indices): + @pytest.mark.parametrize("klass", [pd.Int64Index, pd.DatetimeIndex]) + def test_truncate_decreasing_index(self, before, after, indices, klass): # https://github.com/pandas-dev/pandas/issues/33756 - idx = pd.Index([3, 2, 1, 0]) + idx = klass([3, 2, 1, 0]) + if klass is pd.DatetimeIndex: + before = pd.Timestamp(before) if before is not None else None + after = pd.Timestamp(after) if after is not None else None + indices = [pd.Timestamp(i) for i in indices] values = pd.Series(range(len(idx)), index=idx) result = values.truncate(before=before, after=after) expected = values.loc[indices]