From 3d24808145168d37a4cc6d5e498933cacf5a92b4 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 20 Aug 2019 17:59:51 -0700 Subject: [PATCH 1/5] BUG: timedelta64(NaT) incorrectly treated as datetime in some dataframe ops --- pandas/core/ops/__init__.py | 13 +++++++++++-- pandas/tests/frame/test_arithmetic.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py index 7e03b9544ee72..892622275541e 100644 --- a/pandas/core/ops/__init__.py +++ b/pandas/core/ops/__init__.py @@ -498,8 +498,17 @@ def column_op(a, b): # in which case we specifically want to operate row-by-row assert right.index.equals(left.columns) - def column_op(a, b): - return {i: func(a.iloc[:, i], b.iloc[i]) for i in range(len(a.columns))} + if right.dtype == "timedelta64[ns]": + # ensure we treat NaT values as the correct dtype + right = np.asarray(right) + + def column_op(a, b): + return {i: func(a.iloc[:, i], b[i]) for i in range(len(a.columns))} + + else: + + def column_op(a, b): + return {i: func(a.iloc[:, i], b.iloc[i]) for i in range(len(a.columns))} elif isinstance(right, ABCSeries): assert right.index.equals(left.index) # Handle other cases later diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py index 706bc122c6d9e..fc3640503e385 100644 --- a/pandas/tests/frame/test_arithmetic.py +++ b/pandas/tests/frame/test_arithmetic.py @@ -457,6 +457,16 @@ def test_arith_flex_zero_len_raises(self): class TestFrameArithmetic: + def test_td64_op_nat_casting(self): + # Make sure we don't accidentally treat timedelta64(NaT) as datetime64 + # when calling dispatch_to_series in DataFrame arithmetic + ser = pd.Series(["NaT", "NaT"], dtype="timedelta64[ns]") + df = pd.DataFrame([[1, 2], [3, 4]]) + + result = df * ser + expected = pd.DataFrame({0: ser, 1: ser}) + tm.assert_frame_equal(result, expected) + def test_df_add_2d_array_rowlike_broadcasts(self): # GH#23000 arr = np.arange(6).reshape(3, 2) From 90b92a0c1c94b1644313cdf810d09dbea7bba083 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 3 Sep 2019 08:14:40 -0700 Subject: [PATCH 2/5] comment --- pandas/core/ops/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py index 9c8d3f945c902..f5cdcf3225c75 100644 --- a/pandas/core/ops/__init__.py +++ b/pandas/core/ops/__init__.py @@ -501,6 +501,8 @@ def column_op(a, b): if right.dtype == "timedelta64[ns]": # ensure we treat NaT values as the correct dtype + # Note: we do not do this unconditionally as it may be lossy or + # expensive for EA dtypes. right = np.asarray(right) def column_op(a, b): From 9c1200133e76a4e27fd3a8d48f3fee00874a4261 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 3 Sep 2019 09:20:57 -0700 Subject: [PATCH 3/5] whatsnew --- doc/source/whatsnew/v0.25.2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.2.rst b/doc/source/whatsnew/v0.25.2.rst index 1cdf213d81a74..c9b2108cea0e9 100644 --- a/doc/source/whatsnew/v0.25.2.rst +++ b/doc/source/whatsnew/v0.25.2.rst @@ -18,7 +18,7 @@ Categorical Datetimelike ^^^^^^^^^^^^ - +- Bug in :class:`DataFrame` arithmetic operations when operating with a :class:`Series` with dtype `'timedelta64[ns]'` (:issue:`28049`) - - - From 86070e18b84d2a831685e2c77132467e11f2599c Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 17 Sep 2019 07:17:04 -0700 Subject: [PATCH 4/5] DOC: move whatsnew note to 1.0 --- doc/source/whatsnew/v0.25.2.rst | 1 - doc/source/whatsnew/v1.0.0.rst | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.2.rst b/doc/source/whatsnew/v0.25.2.rst index a92b6f7ca825a..c65240a8d844f 100644 --- a/doc/source/whatsnew/v0.25.2.rst +++ b/doc/source/whatsnew/v0.25.2.rst @@ -18,7 +18,6 @@ Categorical Datetimelike ^^^^^^^^^^^^ -- Bug in :class:`DataFrame` arithmetic operations when operating with a :class:`Series` with dtype `'timedelta64[ns]'` (:issue:`28049`) - - - diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 8b2b3a09f8c87..f08037526dedd 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -136,6 +136,7 @@ Datetimelike - Bug in :class:`Timestamp` subtraction when subtracting a :class:`Timestamp` from a ``np.datetime64`` object incorrectly raising ``TypeError`` (:issue:`28286`) - Addition and subtraction of integer or integer-dtype arrays with :class:`Timestamp` will now raise ``NullFrequencyError`` instead of ``ValueError`` (:issue:`28268`) - Bug in :class:`Series` and :class:`DataFrame` with integer dtype failing to raise ``TypeError`` when adding or subtracting a ``np.datetime64`` object (:issue:`28080`) +- Bug in :class:`DataFrame` arithmetic operations when operating with a :class:`Series` with dtype `'timedelta64[ns]'` (:issue:`28049`) - From e3514e438e403ea6d69edfdc71af6287a29b3c36 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Wed, 18 Sep 2019 08:45:11 -0700 Subject: [PATCH 5/5] reverted whitespace in 0.25 whatsnew --- doc/source/whatsnew/v0.25.2.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.25.2.rst b/doc/source/whatsnew/v0.25.2.rst index c65240a8d844f..de411ef63680a 100644 --- a/doc/source/whatsnew/v0.25.2.rst +++ b/doc/source/whatsnew/v0.25.2.rst @@ -18,6 +18,7 @@ Categorical Datetimelike ^^^^^^^^^^^^ + - - -