From 05b2c504f27cd77077d3149b3d0d6fec13020058 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 09:53:25 -0700 Subject: [PATCH 01/11] un-xfail tests fixed by #22163 --- pandas/tests/test_arithmetic.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pandas/tests/test_arithmetic.py b/pandas/tests/test_arithmetic.py index ae067faba7929..943c5192d7316 100644 --- a/pandas/tests/test_arithmetic.py +++ b/pandas/tests/test_arithmetic.py @@ -865,9 +865,6 @@ def test_td64arr_sub_NaT(self, box): def test_td64arr_add_timedeltalike(self, delta, box): # only test adding/sub offsets as + is now numeric - if box is pd.DataFrame and isinstance(delta, pd.DateOffset): - pytest.xfail(reason="Returns object dtype instead of m8[ns]") - rng = timedelta_range('1 days', '10 days') expected = timedelta_range('1 days 02:00:00', '10 days 02:00:00', freq='D') @@ -879,9 +876,6 @@ def test_td64arr_add_timedeltalike(self, delta, box): def test_td64arr_sub_timedeltalike(self, delta, box): # only test adding/sub offsets as - is now numeric - if box is pd.DataFrame and isinstance(delta, pd.DateOffset): - pytest.xfail(reason="Returns object dtype instead of m8[ns]") - rng = timedelta_range('1 days', '10 days') expected = timedelta_range('0 days 22:00:00', '9 days 22:00:00') @@ -1132,9 +1126,6 @@ def test_td64arr_mul_int(self, box): tm.assert_equal(result, idx) def test_td64arr_mul_tdlike_scalar_raises(self, delta, box): - if box is pd.DataFrame and not isinstance(delta, pd.DateOffset): - pytest.xfail(reason="returns m8[ns] instead of raising") - rng = timedelta_range('1 days', '10 days', name='foo') rng = tm.box_expected(rng, box) with pytest.raises(TypeError): From 6dbb2e639aa7535a7ccd55082e6242911f3594b7 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 10:31:59 -0700 Subject: [PATCH 02/11] Fix Index[object] op with Series[timedelta64] --- doc/source/whatsnew/v0.24.0.txt | 2 +- pandas/core/indexes/base.py | 2 ++ pandas/tests/test_arithmetic.py | 6 +----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index cf12759c051fc..9b12c595c6bba 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -576,7 +576,7 @@ Datetimelike - Bug in :class:`DataFrame` with mixed dtypes including ``datetime64[ns]`` incorrectly raising ``TypeError`` on equality comparisons (:issue:`13128`,:issue:`22163`) - Bug in :meth:`DataFrame.eq` comparison against ``NaT`` incorrectly returning ``True`` or ``NaN`` (:issue:`15697`,:issue:`22163`) - Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`) -- +- Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`?????`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index bfa669a0ca164..8eb2016e0976f 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -2645,6 +2645,8 @@ def argsort(self, *args, **kwargs): return result.argsort(*args, **kwargs) def __add__(self, other): + if isinstance(other, (ABCSeries, ABCDataFrame)): + return NotImplemented return Index(np.array(self) + other) def __radd__(self, other): diff --git a/pandas/tests/test_arithmetic.py b/pandas/tests/test_arithmetic.py index 943c5192d7316..e8fd471d474f1 100644 --- a/pandas/tests/test_arithmetic.py +++ b/pandas/tests/test_arithmetic.py @@ -923,11 +923,7 @@ def test_timedelta64_operations_with_DateOffset(self): @pytest.mark.parametrize('box', [ pd.Index, - pytest.param(Series, - marks=pytest.mark.xfail(reason="Index fails to return " - "NotImplemented on " - "reverse op", - strict=True)), + Series, pytest.param(pd.DataFrame, marks=pytest.mark.xfail(reason="Tries to broadcast " "incorrectly", From 8c3df15102d80c45a3d67d8fc61405ffdefabf4e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 10:32:52 -0700 Subject: [PATCH 03/11] Fix multiplication of timedelta with Series --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/ops.py | 6 ++++++ pandas/tests/arithmetic/test_numeric.py | 5 ----- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 9b12c595c6bba..e148be3f6a2c5 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -577,6 +577,7 @@ Datetimelike - Bug in :meth:`DataFrame.eq` comparison against ``NaT`` incorrectly returning ``True`` or ``NaN`` (:issue:`15697`,:issue:`22163`) - Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`) - Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`?????`) +- Bug in multiplying a :class:`Series` with numeric dtype against a ``timedelta`` object (:issue:`?????`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 10418ccbb1f64..d4620b7f953be 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -1244,6 +1244,12 @@ def wrapper(left, right): index=left.index, name=res_name, dtype=result.dtype) + elif type(right) is datetime.timedelta: + # cast up to Timedelta to rely on Timedelta implementation; + # otherwise operation against numeric-dtype raises TypeError + right = pd.Timedelta(right) + return op(left, right) + lvalues = left.values rvalues = right if isinstance(rvalues, ABCSeries): diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index 85a0a8dffc55f..34b8e29c57da0 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -71,11 +71,6 @@ def test_ops_series(self): ids=lambda x: type(x).__name__) def test_numeric_arr_mul_tdscalar(self, scalar_td, index, box): # GH#19333 - - if (box in [Series, pd.DataFrame] and - type(scalar_td) is timedelta and index.dtype == 'f8'): - raise pytest.xfail(reason="Cannot multiply timedelta by float") - expected = pd.timedelta_range('1 days', '10 days') index = tm.box_expected(index, box) From a7502ea86f85e4a9f64ae4298536b8ea2641c3ea Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 10:34:52 -0700 Subject: [PATCH 04/11] un-xfail no-longer-broken tests --- pandas/tests/arithmetic/test_numeric.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index 34b8e29c57da0..c77c4d38fd0e3 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -94,12 +94,6 @@ def test_numeric_arr_mul_tdscalar(self, scalar_td, index, box): Timedelta(days=1).to_pytimedelta()], ids=lambda x: type(x).__name__) def test_numeric_arr_rdiv_tdscalar(self, scalar_td, index, box): - - if box is Series and type(scalar_td) is timedelta: - raise pytest.xfail(reason="TODO: Figure out why this case fails") - if box is pd.DataFrame and isinstance(scalar_td, timedelta): - raise pytest.xfail(reason="TODO: Figure out why this case fails") - expected = TimedeltaIndex(['1 Day', '12 Hours']) index = tm.box_expected(index, box) From ef14164fd7988ca2772d042523015c594df622e9 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 12:07:12 -0700 Subject: [PATCH 05/11] Fix numeric Series add/sub timedelta64array/index --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/ops.py | 19 ++++++++++++ pandas/tests/test_arithmetic.py | 53 +++++++-------------------------- 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index e148be3f6a2c5..b8970d7aa889d 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -578,6 +578,7 @@ Datetimelike - Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`) - Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`?????`) - Bug in multiplying a :class:`Series` with numeric dtype against a ``timedelta`` object (:issue:`?????`) +- Bug in :class:`Series` with numeric dtype when adding or subtracting an an array or ``Series`` with ``timedelta64`` dtype (:issue:`?????`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/ops.py b/pandas/core/ops.py index d4620b7f953be..5944a496c4a30 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -1244,6 +1244,25 @@ def wrapper(left, right): index=left.index, name=res_name, dtype=result.dtype) + elif is_timedelta64_dtype(right) and not is_scalar(right): + # i.e. exclude np.timedelta64 object + # Unfortunately we need to special-case right-hand timedelta64 + # dtypes because numpy casts integer dtypes to timedelta64 when + # operating with timedelta64 + if isinstance(right, np.ndarray): + # upcast to TimedeltaIndex before dispatching + right = pd.TimedeltaIndex(right) + right.freq = None + # TODO: Should we be treating zero-dim ndarray as scalar? + + # Note: we cannot use dispatch_to_index_op because + # that may incorrectly raise TypeError when we + # should get NullFrequencyError + result = op(pd.Index(left), right) + return construct_result(left, result, + index=left.index, name=res_name, + dtype=result.dtype) + elif type(right) is datetime.timedelta: # cast up to Timedelta to rely on Timedelta implementation; # otherwise operation against numeric-dtype raises TypeError diff --git a/pandas/tests/test_arithmetic.py b/pandas/tests/test_arithmetic.py index e8fd471d474f1..def7a8be95fc8 100644 --- a/pandas/tests/test_arithmetic.py +++ b/pandas/tests/test_arithmetic.py @@ -573,19 +573,8 @@ def test_td64arr_add_int_series_invalid(self, box, tdser): with pytest.raises(err): tdser + Series([2, 3, 4]) - @pytest.mark.parametrize('box', [ - pd.Index, - pytest.param(Series, - marks=pytest.mark.xfail(reason="GH#19123 integer " - "interpreted as " - "nanoseconds", - strict=True)), - pytest.param(pd.DataFrame, - marks=pytest.mark.xfail(reason="Attempts to broadcast " - "incorrectly", - strict=True, raises=ValueError)) - ], ids=lambda x: x.__name__) - def test_td64arr_radd_int_series_invalid(self, box, tdser): + def test_td64arr_radd_int_series_invalid(self, box_df_fail, tdser): + box = box_df_fail # Tries to broadcast incorrectly tdser = tm.box_expected(tdser, box) err = TypeError if box is not pd.Index else NullFrequencyError with pytest.raises(err): @@ -605,11 +594,11 @@ def test_td64arr_sub_int_series_invalid(self, box, tdser): with pytest.raises(err): tdser - Series([2, 3, 4]) - @pytest.mark.xfail(reason='GH#19123 integer interpreted as nanoseconds', - strict=True) - def test_td64arr_rsub_int_series_invalid(self, box, tdser): + def test_td64arr_rsub_int_series_invalid(self, box_df_fail, tdser): + box = box_df_fail # Tries to broadcast incorrectly tdser = tm.box_expected(tdser, box) - with pytest.raises(TypeError): + err = TypeError if box is not pd.Index else NullFrequencyError + with pytest.raises(err): Series([2, 3, 4]) - tdser @pytest.mark.parametrize('box', [ @@ -671,14 +660,6 @@ def test_td64arr_add_sub_numeric_scalar_invalid(self, box, scalar, tdser): with pytest.raises(err): scalar - tdser - @pytest.mark.parametrize('box', [ - pd.Index, - Series, - pytest.param(pd.DataFrame, - marks=pytest.mark.xfail(reason="Tries to broadcast " - "incorrectly", - strict=True, raises=ValueError)) - ], ids=lambda x: x.__name__) @pytest.mark.parametrize('dtype', ['int64', 'int32', 'int16', 'uint64', 'uint32', 'uint16', 'uint8', 'float64', 'float32', 'float16']) @@ -688,10 +669,9 @@ def test_td64arr_add_sub_numeric_scalar_invalid(self, box, scalar, tdser): Series([1, 2, 3]) # TODO: Add DataFrame in here? ], ids=lambda x: type(x).__name__) - def test_td64arr_add_sub_numeric_arr_invalid(self, box, vec, dtype, tdser): - if type(vec) is Series and not dtype.startswith('float'): - pytest.xfail(reason='GH#19123 integer interpreted as nanos') - + def test_td64arr_add_sub_numeric_arr_invalid(self, box_df_fail, vec, + dtype, tdser): + box = box_df_fail # tries to broadcast incorrectly tdser = tm.box_expected(tdser, box) err = TypeError if box is pd.Index and not dtype.startswith('float'): @@ -1011,23 +991,12 @@ def test_td64arr_sub_offset_array(self, box_df_fail): res = tdi - other tm.assert_equal(res, expected) - @pytest.mark.parametrize('box', [ - pd.Index, - pytest.param(Series, - marks=pytest.mark.xfail(reason="object dtype Series " - "fails to return " - "NotImplemented", - strict=True, raises=TypeError)), - pytest.param(pd.DataFrame, - marks=pytest.mark.xfail(reason="tries to broadcast " - "incorrectly", - strict=True, raises=ValueError)) - ], ids=lambda x: x.__name__) @pytest.mark.parametrize('names', [(None, None, None), ('foo', 'bar', None), ('foo', 'foo', 'foo')]) - def test_td64arr_with_offset_series(self, names, box): + def test_td64arr_with_offset_series(self, names, box_df_fail): # GH#18849 + box = box_df_fail # tries to broadcast incorrectly box2 = Series if box is pd.Index else box tdi = TimedeltaIndex(['1 days 00:00:00', '3 days 04:00:00'], From 6b026911e05e28a1248c4f377b48e0a25e02979d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 13:17:40 -0700 Subject: [PATCH 06/11] Fix/test numeric index ops with timedelta64 array/index/series --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/indexes/base.py | 8 ++++ pandas/core/indexes/range.py | 4 ++ pandas/tests/arithmetic/test_numeric.py | 54 ++++++++++++++++++++++++- 4 files changed, 66 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index b8970d7aa889d..e8476cec5ede4 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -579,6 +579,7 @@ Datetimelike - Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`?????`) - Bug in multiplying a :class:`Series` with numeric dtype against a ``timedelta`` object (:issue:`?????`) - Bug in :class:`Series` with numeric dtype when adding or subtracting an an array or ``Series`` with ``timedelta64`` dtype (:issue:`?????`) +- Bug in :class:`Index` with numeric dtype when multiplying or dividing an array with dtype ``timedelta64`` (:issue:`????`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 8eb2016e0976f..52105b3028f08 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -122,6 +122,14 @@ def index_arithmetic_method(self, other): elif isinstance(other, ABCTimedeltaIndex): # Defer to subclass implementation return NotImplemented + elif isinstance(other, np.ndarray) and is_timedelta64_dtype(other): + # wrap in Series for op, this will in turn wrap in TimedeltaIndex, + # but will correctly raise TypeError instead of NullFrequencyError + # for add/sub ops + from pandas import Series + other = Series(other) + out = op(self, other) + return Index(out, name=self.name) other = self._validate_for_numeric_binop(other, op) diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 939ec0b79ac6b..8713300ea5635 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -8,6 +8,7 @@ from pandas.core.dtypes.common import ( is_integer, is_scalar, + is_timedelta64_dtype, is_int64_dtype) from pandas.core.dtypes.generic import ABCSeries, ABCTimedeltaIndex @@ -596,6 +597,9 @@ def _evaluate_numeric_binop(self, other): # GH#19333 is_integer evaluated True on timedelta64, # so we need to catch these explicitly return op(self._int64index, other) + elif is_timedelta64_dtype(other): + # Must be an np.ndarray + return op(self._int64index, other) other = self._validate_for_numeric_binop(other, op) attrs = self._get_attributes_dict() diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index c77c4d38fd0e3..507f3e6f2142a 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -47,7 +47,59 @@ def test_operator_series_comparison_zerorank(self): # ------------------------------------------------------------------ # Numeric dtypes Arithmetic with Timedelta Scalar -class TestNumericArraylikeArithmeticWithTimedeltaScalar(object): +class TestNumericArraylikeArithmeticWithTimedeltaLike(object): + + # TODO: also check name retentention + @pytest.mark.parametrize('box_cls', [np.array, pd.Index, pd.Series]) + @pytest.mark.parametrize('left', [ + pd.RangeIndex(10, 40, 10)] + [cls([10, 20, 30], dtype=dtype) + for dtype in ['i1', 'i2', 'i4', 'i8', + 'u1', 'u2', 'u4', 'u8', + 'f2', 'f4', 'f8'] + for cls in [pd.Series, pd.Index] + ], ids=lambda x: type(x).__name__ + str(x.dtype)) + def test_mul_td64arr(self, left, box_cls): + right = np.array([1, 2, 3], dtype='m8[s]') + right = box_cls(right) + + expected = pd.TimedeltaIndex(['10s', '40s', '90s']) + if isinstance(left, pd.Series) or box_cls is pd.Series: + expected = pd.Series(expected) + + result = left * right + tm.assert_equal(result, expected) + + result = right * left + tm.assert_equal(result, expected) + + # TODO: also check name retentention + @pytest.mark.parametrize('box_cls', [np.array, pd.Index, pd.Series]) + @pytest.mark.parametrize('left', [ + pd.RangeIndex(10, 40, 10)] + [cls([10, 20, 30], dtype=dtype) + for dtype in ['i1', 'i2', 'i4', 'i8', + 'u1', 'u2', 'u4', 'u8', + 'f2', 'f4', 'f8'] + for cls in [pd.Series, pd.Index] + ], ids=lambda x: type(x).__name__ + str(x.dtype)) + def test_div_td64arr(self, left, box_cls): + right = np.array([10, 40, 90], dtype='m8[s]') + right = box_cls(right) + + expected = pd.TimedeltaIndex(['1s', '2s', '3s']) + if isinstance(left, pd.Series) or box_cls is pd.Series: + expected = pd.Series(expected) + + result = right / left + tm.assert_equal(result, expected) + + result = right // left + tm.assert_equal(result, expected) + + with pytest.raises(TypeError): + left / right + + with pytest.raises(TypeError): + left // right # TODO: de-duplicate with test_numeric_arr_mul_tdscalar def test_ops_series(self): From b856744b3f0b72b1fdf2a30443c8f8a0dfffdacf Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 13:26:19 -0700 Subject: [PATCH 07/11] flake8 fixup --- pandas/tests/arithmetic/test_numeric.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index 507f3e6f2142a..19e65b6264572 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -56,8 +56,8 @@ class TestNumericArraylikeArithmeticWithTimedeltaLike(object): for dtype in ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8', 'f2', 'f4', 'f8'] - for cls in [pd.Series, pd.Index] - ], ids=lambda x: type(x).__name__ + str(x.dtype)) + for cls in [pd.Series, pd.Index]], + ids=lambda x: type(x).__name__ + str(x.dtype)) def test_mul_td64arr(self, left, box_cls): right = np.array([1, 2, 3], dtype='m8[s]') right = box_cls(right) @@ -79,8 +79,8 @@ def test_mul_td64arr(self, left, box_cls): for dtype in ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8', 'f2', 'f4', 'f8'] - for cls in [pd.Series, pd.Index] - ], ids=lambda x: type(x).__name__ + str(x.dtype)) + for cls in [pd.Series, pd.Index]], + ids=lambda x: type(x).__name__ + str(x.dtype)) def test_div_td64arr(self, left, box_cls): right = np.array([10, 40, 90], dtype='m8[s]') right = box_cls(right) From c493b237709c995c6e17cd7fd0bd6d102321f624 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 13:37:20 -0700 Subject: [PATCH 08/11] add GH references --- doc/source/whatsnew/v0.24.0.txt | 8 ++++---- pandas/core/indexes/base.py | 6 +++--- pandas/core/indexes/range.py | 2 +- pandas/core/ops.py | 11 ++++++----- pandas/tests/arithmetic/test_numeric.py | 2 ++ 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index e8476cec5ede4..04fcf03b2f5ee 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -576,10 +576,10 @@ Datetimelike - Bug in :class:`DataFrame` with mixed dtypes including ``datetime64[ns]`` incorrectly raising ``TypeError`` on equality comparisons (:issue:`13128`,:issue:`22163`) - Bug in :meth:`DataFrame.eq` comparison against ``NaT`` incorrectly returning ``True`` or ``NaN`` (:issue:`15697`,:issue:`22163`) - Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`) -- Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`?????`) -- Bug in multiplying a :class:`Series` with numeric dtype against a ``timedelta`` object (:issue:`?????`) -- Bug in :class:`Series` with numeric dtype when adding or subtracting an an array or ``Series`` with ``timedelta64`` dtype (:issue:`?????`) -- Bug in :class:`Index` with numeric dtype when multiplying or dividing an array with dtype ``timedelta64`` (:issue:`????`) +- Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`22390`) +- Bug in multiplying a :class:`Series` with numeric dtype against a ``timedelta`` object (:issue:`22390`) +- Bug in :class:`Series` with numeric dtype when adding or subtracting an an array or ``Series`` with ``timedelta64`` dtype (:issue:`22390`) +- Bug in :class:`Index` with numeric dtype when multiplying or dividing an array with dtype ``timedelta64`` (:issue:`22390`) Timedelta ^^^^^^^^^ diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 52105b3028f08..164887f7501c9 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -123,9 +123,9 @@ def index_arithmetic_method(self, other): # Defer to subclass implementation return NotImplemented elif isinstance(other, np.ndarray) and is_timedelta64_dtype(other): - # wrap in Series for op, this will in turn wrap in TimedeltaIndex, - # but will correctly raise TypeError instead of NullFrequencyError - # for add/sub ops + # GH#22390; wrap in Series for op, this will in turn wrap in + # TimedeltaIndex, but will correctly raise TypeError instead of + # NullFrequencyError for add/sub ops from pandas import Series other = Series(other) out = op(self, other) diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 8713300ea5635..981bfddeadac1 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -598,7 +598,7 @@ def _evaluate_numeric_binop(self, other): # so we need to catch these explicitly return op(self._int64index, other) elif is_timedelta64_dtype(other): - # Must be an np.ndarray + # Must be an np.ndarray; GH#22390 return op(self._int64index, other) other = self._validate_for_numeric_binop(other, op) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 5944a496c4a30..3fe8f7e3c88d6 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -1246,9 +1246,9 @@ def wrapper(left, right): elif is_timedelta64_dtype(right) and not is_scalar(right): # i.e. exclude np.timedelta64 object - # Unfortunately we need to special-case right-hand timedelta64 - # dtypes because numpy casts integer dtypes to timedelta64 when - # operating with timedelta64 + # GH#22390 Unfortunately we need to special-case right-hand + # timedelta64 dtypes because numpy casts integer dtypes to + # timedelta64 when operating with timedelta64 if isinstance(right, np.ndarray): # upcast to TimedeltaIndex before dispatching right = pd.TimedeltaIndex(right) @@ -1264,8 +1264,9 @@ def wrapper(left, right): dtype=result.dtype) elif type(right) is datetime.timedelta: - # cast up to Timedelta to rely on Timedelta implementation; - # otherwise operation against numeric-dtype raises TypeError + # GH#22390 cast up to Timedelta to rely on Timedelta + # implementation; otherwise operation against numeric-dtype + # raises TypeError right = pd.Timedelta(right) return op(left, right) diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index 19e65b6264572..64955437b21ec 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -59,6 +59,7 @@ class TestNumericArraylikeArithmeticWithTimedeltaLike(object): for cls in [pd.Series, pd.Index]], ids=lambda x: type(x).__name__ + str(x.dtype)) def test_mul_td64arr(self, left, box_cls): + # GH#22390 right = np.array([1, 2, 3], dtype='m8[s]') right = box_cls(right) @@ -82,6 +83,7 @@ def test_mul_td64arr(self, left, box_cls): for cls in [pd.Series, pd.Index]], ids=lambda x: type(x).__name__ + str(x.dtype)) def test_div_td64arr(self, left, box_cls): + # GH#22390 right = np.array([10, 40, 90], dtype='m8[s]') right = box_cls(right) From 000e6dfbf1a29f60129229c388de3198f052dfac Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 16 Aug 2018 14:32:06 -0700 Subject: [PATCH 09/11] Fixup remove unused import --- pandas/tests/arithmetic/test_numeric.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/arithmetic/test_numeric.py b/pandas/tests/arithmetic/test_numeric.py index 64955437b21ec..9ede1a62aaf2e 100644 --- a/pandas/tests/arithmetic/test_numeric.py +++ b/pandas/tests/arithmetic/test_numeric.py @@ -2,7 +2,6 @@ # Arithmetc tests for DataFrame/Series/Index/Array classes that should # behave identically. # Specifically for numeric dtypes -from datetime import timedelta from decimal import Decimal import operator from collections import Iterable From 6bd5d0ec0120933b6b4ed6e1d9480892e93aa51c Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 20 Aug 2018 09:22:21 -0700 Subject: [PATCH 10/11] implement maybe_upcast_for_op --- pandas/core/ops.py | 48 ++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 5c5959d002080..b25809bf074f7 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -107,6 +107,37 @@ def _maybe_match_name(a, b): return None +def maybe_upcast_for_op(obj): + """ + Cast non-pandas objects to pandas types to unify behavior of arithmetic + and comparison operations. + + Parameters + ---------- + obj: object + + Returns + ------- + out : object + + Notes + ----- + Be careful to call this *after* determining the `name` attribute to be + attached to the result of the arithmetic operation. + """ + if type(obj) is datetime.timedelta: + # GH#22390 cast up to Timedelta to rely on Timedelta + # implementation; otherwise operation against numeric-dtype + # raises TypeError + return pd.Timedelta(obj) + elif isinstance(obj, np.ndarray) and is_timedelta64_dtype(obj): + # GH#22390 Unfortunately we need to special-case right-hand + # timedelta64 dtypes because numpy casts integer dtypes to + # timedelta64 when operating with timedelta64 + return pd.TimedeltaIndex(obj) + return obj + + # ----------------------------------------------------------------------------- # Reversed Operations not available in the stdlib operator module. # Defining these instead of using lambdas allows us to reference them by name. @@ -1222,6 +1253,7 @@ def wrapper(left, right): left, right = _align_method_SERIES(left, right) res_name = get_op_result_name(left, right) + right = maybe_upcast_for_op(right) if is_categorical_dtype(left): raise TypeError("{typ} cannot perform the operation " @@ -1246,15 +1278,6 @@ def wrapper(left, right): elif is_timedelta64_dtype(right) and not is_scalar(right): # i.e. exclude np.timedelta64 object - # GH#22390 Unfortunately we need to special-case right-hand - # timedelta64 dtypes because numpy casts integer dtypes to - # timedelta64 when operating with timedelta64 - if isinstance(right, np.ndarray): - # upcast to TimedeltaIndex before dispatching - right = pd.TimedeltaIndex(right) - right.freq = None - # TODO: Should we be treating zero-dim ndarray as scalar? - # Note: we cannot use dispatch_to_index_op because # that may incorrectly raise TypeError when we # should get NullFrequencyError @@ -1263,13 +1286,6 @@ def wrapper(left, right): index=left.index, name=res_name, dtype=result.dtype) - elif type(right) is datetime.timedelta: - # GH#22390 cast up to Timedelta to rely on Timedelta - # implementation; otherwise operation against numeric-dtype - # raises TypeError - right = pd.Timedelta(right) - return op(left, right) - lvalues = left.values rvalues = right if isinstance(rvalues, ABCSeries): From cdfb3bfceb00f33f66c191d4032644e137a1eec8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 22 Aug 2018 06:52:56 -0700 Subject: [PATCH 11/11] move notes to Timedelta section --- doc/source/whatsnew/v0.24.0.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 5f8fa561a5e41..e87e2d356d393 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -581,15 +581,14 @@ Datetimelike - Bug in :class:`DataFrame` comparisons against ``Timestamp``-like objects failing to raise ``TypeError`` for inequality checks with mismatched types (:issue:`8932`,:issue:`22163`) - Bug in :class:`DataFrame` with mixed dtypes including ``datetime64[ns]`` incorrectly raising ``TypeError`` on equality comparisons (:issue:`13128`,:issue:`22163`) - Bug in :meth:`DataFrame.eq` comparison against ``NaT`` incorrectly returning ``True`` or ``NaN`` (:issue:`15697`,:issue:`22163`) + +Timedelta +^^^^^^^^^ - Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`) - Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`22390`) - Bug in multiplying a :class:`Series` with numeric dtype against a ``timedelta`` object (:issue:`22390`) - Bug in :class:`Series` with numeric dtype when adding or subtracting an an array or ``Series`` with ``timedelta64`` dtype (:issue:`22390`) - Bug in :class:`Index` with numeric dtype when multiplying or dividing an array with dtype ``timedelta64`` (:issue:`22390`) - -Timedelta -^^^^^^^^^ - - - -