From 864c32b5aad502f4d03e35da147b4b79122c4f24 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 8 Jul 2019 19:09:30 -0700 Subject: [PATCH 1/5] BUG: Fix inserting TImedelta into Series, closes #22717 --- pandas/core/indexing.py | 21 +++++++++++++++---- pandas/tests/series/indexing/test_indexing.py | 9 ++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index e27f85eb6d0a4..235801d0a9359 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -19,6 +19,7 @@ is_scalar, is_sequence, is_sparse, + is_timedelta64_dtype, ) from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries from pandas.core.dtypes.missing import _infer_fill_value, isna @@ -429,11 +430,23 @@ def _setitem_with_indexer(self, indexer, value): # this preserves dtype of the value new_values = Series([value])._values if len(self.obj._values): - try: - new_values = np.concatenate([self.obj._values, new_values]) - except TypeError: + if is_timedelta64_dtype( + new_values + ) and not is_timedelta64_dtype(self.obj): + # GH#22717 np.concatenate incorrect casts + # timedelta64 to integer as_obj = self.obj.astype(object) - new_values = np.concatenate([as_obj, new_values]) + new_values = np.concatenate( + [as_obj, np.array([value], dtype=object)] + ) + else: + try: + new_values = np.concatenate( + [self.obj._values, new_values] + ) + except TypeError: + as_obj = self.obj.astype(object) + new_values = np.concatenate([as_obj, new_values]) self.obj._data = self.obj._constructor( new_values, index=new_index, name=self.obj.name )._data diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index c8342c54e9b5d..7ffbbbd6125c5 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -653,6 +653,15 @@ def test_timedelta_assignment(): expected.loc[[1, 2, 3]] = pd.Timedelta(np.timedelta64(20, "m")) tm.assert_series_equal(s, expected) + # GH#22717 inserting a Timedelta should _not_ cast to int64 + ser = pd.Series(["x"]) + ser["td"] = pd.Timedelta("9 days") + assert isinstance(ser["td"], pd.Timedelta) + + ser = pd.Series(["x"]) + ser.loc["td"] = pd.Timedelta("9 days") + assert isinstance(ser["td"], pd.Timedelta) + def test_underlying_data_conversion(): # GH 4080 From 82cfca445299c80e6f809ca2fac11b779956fb07 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 8 Jul 2019 19:25:32 -0700 Subject: [PATCH 2/5] Whatsnew --- doc/source/whatsnew/v0.25.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 68ecb4c487a1e..83e2756e0ec34 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -1050,7 +1050,7 @@ Indexing - Bug which produced ``AttributeError`` on partial matching :class:`Timestamp` in a :class:`MultiIndex` (:issue:`26944`) - Bug in :class:`Categorical` and :class:`CategoricalIndex` with :class:`Interval` values when using the ``in`` operator (``__contains``) with objects that are not comparable to the values in the ``Interval`` (:issue:`23705`) - Bug in :meth:`DataFrame.loc` and :meth:`DataFrame.iloc` on a :class:`DataFrame` with a single timezone-aware datetime64[ns] column incorrectly returning a scalar instead of a :class:`Series` (:issue:`27110`) -- +- Bug in setting a new value in a :class:`Series` with a :class:`Timedelta` object incorrectly casting the value to an integer (:issue:`22717`) Missing ^^^^^^^ From 2c6d922f96308f73c3a7823079c667b499442067 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 9 Jul 2019 18:19:41 -0700 Subject: [PATCH 3/5] separate+parametrize test --- pandas/tests/series/indexing/test_indexing.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 7ffbbbd6125c5..908807ff109bf 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -653,13 +653,29 @@ def test_timedelta_assignment(): expected.loc[[1, 2, 3]] = pd.Timedelta(np.timedelta64(20, "m")) tm.assert_series_equal(s, expected) + +@pytest.mark.parametrize( + "td", + [ + pd.Timedelta("9 days"), + pd.Timedelta("9 days").to_timedelta64(), + pd.Timedelta("9 days").to_pytimedelta(), + ], +) +def test_append_timedelta_does_not_cast(td): # GH#22717 inserting a Timedelta should _not_ cast to int64 + expected = pd.Series(["x", td], index=[0, "td"], dtype=object) + + # FIXME: these should either _both_ cast the object to Timedelta + # or both retain type(td) ser = pd.Series(["x"]) - ser["td"] = pd.Timedelta("9 days") - assert isinstance(ser["td"], pd.Timedelta) + ser["td"] = td + tm.assert_series_equal(ser, expected) + assert isinstance(ser["td"], type(td)) ser = pd.Series(["x"]) ser.loc["td"] = pd.Timedelta("9 days") + tm.assert_series_equal(ser, expected) assert isinstance(ser["td"], pd.Timedelta) From d1b06fc3e351a5c01d88f34c5193c89679dcbf1c Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 9 Jul 2019 19:29:48 -0700 Subject: [PATCH 4/5] use concat_compat --- pandas/core/indexing.py | 22 ++++--------------- pandas/tests/series/indexing/test_indexing.py | 2 +- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index c3058c5315cad..5f0473e43382b 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -19,8 +19,8 @@ is_scalar, is_sequence, is_sparse, - is_timedelta64_dtype, ) +from pandas.core.dtypes.concat import _concat_compat from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries from pandas.core.dtypes.missing import _infer_fill_value, isna @@ -430,23 +430,9 @@ def _setitem_with_indexer(self, indexer, value): # this preserves dtype of the value new_values = Series([value])._values if len(self.obj._values): - if is_timedelta64_dtype( - new_values - ) and not is_timedelta64_dtype(self.obj): - # GH#22717 np.concatenate incorrect casts - # timedelta64 to integer - as_obj = self.obj.astype(object) - new_values = np.concatenate( - [as_obj, np.array([value], dtype=object)] - ) - else: - try: - new_values = np.concatenate( - [self.obj._values, new_values] - ) - except TypeError: - as_obj = self.obj.astype(object) - new_values = np.concatenate([as_obj, new_values]) + # GH#22717 np.concatenate incorrect casts + # timedelta64 to integer + new_values = _concat_compat([self.obj._values, new_values]) self.obj._data = self.obj._constructor( new_values, index=new_index, name=self.obj.name )._data diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 908807ff109bf..93c279cd36e62 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -671,7 +671,7 @@ def test_append_timedelta_does_not_cast(td): ser = pd.Series(["x"]) ser["td"] = td tm.assert_series_equal(ser, expected) - assert isinstance(ser["td"], type(td)) + assert isinstance(ser["td"], pd.Timedelta) ser = pd.Series(["x"]) ser.loc["td"] = pd.Timedelta("9 days") From 4731aaf018e3ac07f9ae38c69478215a4ee036a7 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 10 Jul 2019 10:21:48 -0700 Subject: [PATCH 5/5] edit/remove comments --- pandas/core/indexing.py | 4 ++-- pandas/tests/series/indexing/test_indexing.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 8331edf4332fa..c31d6538ad2c3 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -433,8 +433,8 @@ def _setitem_with_indexer(self, indexer, value): # this preserves dtype of the value new_values = Series([value])._values if len(self.obj._values): - # GH#22717 np.concatenate incorrect casts - # timedelta64 to integer + # GH#22717 handle casting compatibility that np.concatenate + # does incorrectly new_values = _concat_compat([self.obj._values, new_values]) self.obj._data = self.obj._constructor( new_values, index=new_index, name=self.obj.name diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index 93c279cd36e62..1fb1dd3bb998a 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -666,8 +666,6 @@ def test_append_timedelta_does_not_cast(td): # GH#22717 inserting a Timedelta should _not_ cast to int64 expected = pd.Series(["x", td], index=[0, "td"], dtype=object) - # FIXME: these should either _both_ cast the object to Timedelta - # or both retain type(td) ser = pd.Series(["x"]) ser["td"] = td tm.assert_series_equal(ser, expected)