BUG: where behaves badly when dtype of self is datetime or timedelta, and dtype of other is not (GH9804)

Author: evanpw

doc/source/whatsnew/v0.16.1.txt

@@ -117,7 +117,6 @@ Bug Fixes
 - Bug in ``read_csv()`` interprets ``index_col=True`` as ``1`` (:issue:`9798`)
 - Bug in index equality comparisons using ``==`` failing on Index/MultiIndex type incompatibility (:issue:`9875`)
 - Bug in which ``SparseDataFrame`` could not take `nan` as a column name (:issue:`8822`)
-- Bug in ``Series.quantile`` on empty Series of type ``Datetime`` or ``Timedelta`` (:issue:`9675`)
 - Bug in ``to_msgpack`` and ``read_msgpack`` zlib and blosc compression support (:issue:`9783`)
 - Bug in unequal comparisons between a ``Series`` of dtype `"category"` and a scalar (e.g. ``Series(Categorical(list("abc"), categories=list("cba"), ordered=True)) > "b"``, which wouldn't use the order of the categories but use the lexicographical order. (:issue:`9848`)
 
@@ -135,10 +134,11 @@ Bug Fixes
 
 
 
+
 - Bug in unequal comparisons between categorical data and a scalar, which was not in the categories (e.g. ``Series(Categorical(list("abc"), ordered=True)) > "d"``. This returned ``False`` for all elements, but now raises a ``TypeError``. Equality comparisons also now return ``False`` for ``==`` and ``True`` for ``!=``. (:issue:`9848`)
 
 - Bug in DataFrame ``__setitem__`` when right hand side is a dictionary (:issue:`9874`)
-
+- Bug in ``where`` when dtype is ``datetime64/timedelta64``, but dtype of other is not (:issue:`9804`)
 - Bug in ``MultiIndex.sortlevel()`` results in unicode level name breaks (:issue:`9875`)
 - Bug in which ``groupby.transform`` incorrectly enforced output dtypes to match input dtypes. (:issue:`9807`)
 

pandas/core/generic.py

@@ -3335,7 +3335,8 @@ def where(self, cond, other=np.nan, inplace=False, axis=None, level=None,
                 except ValueError:
                     new_other = np.array(other)
 
-                if not (new_other == np.array(other)).all():
+                matches = (new_other == np.array(other))
+                if matches is False or not matches.all():
                     other = np.array(other)
 
                     # we can't use our existing dtype

pandas/core/internals.py

@@ -1325,13 +1325,11 @@ def _try_fill(self, value):
         return value
 
     def _try_coerce_args(self, values, other):
-        """ provide coercion to our input arguments
-            we are going to compare vs i8, so coerce to floats
-            repring NaT with np.nan so nans propagate
-            values is always ndarray like, other may not be """
+        """ Coerce values and other to float64, with null values converted to
+            NaN. values is always ndarray-like, other may not be """
         def masker(v):
             mask = isnull(v)
-            v = v.view('i8').astype('float64')
+            v = v.astype('float64')
             v[mask] = np.nan
             return v
 
@@ -1343,6 +1341,8 @@ def masker(v):
             other = _coerce_scalar_to_timedelta_type(other, unit='s', box=False).item()
             if other == tslib.iNaT:
                 other = np.nan
+        elif lib.isscalar(other):
+            other = np.float64(other)
         else:
             other = masker(other)
 
@@ -1809,16 +1809,20 @@ def _try_operate(self, values):
         return values.view('i8')
 
     def _try_coerce_args(self, values, other):
-        """ provide coercion to our input arguments
-            we are going to compare vs i8, so coerce to integer
-            values is always ndarra like, other may not be """
+        """ Coerce values and other to dtype 'i8'. NaN and NaT convert to
+            the smallest i8, and will correctly round-trip to NaT if converted
+            back in _try_coerce_result. values is always ndarray-like, other
+            may not be """
         values = values.view('i8')
+
         if is_null_datelike_scalar(other):
             other = tslib.iNaT
         elif isinstance(other, datetime):
             other = lib.Timestamp(other).asm8.view('i8')
-        else:
+        elif hasattr(other, 'dtype') and com.is_integer_dtype(other):
             other = other.view('i8')
+        else:
+            other = np.array(other, dtype='i8')
 
         return values, other
 

pandas/tests/test_series.py

@@ -1859,6 +1859,48 @@ def test_where_dups(self):
         expected = Series([5,11,2,5,11,2],index=[0,1,2,0,1,2])
         assert_series_equal(comb, expected)
 
+    def test_where_datetime(self):
+        s = Series(date_range('20130102', periods=2))
+        expected = Series([10, 10], dtype='datetime64[ns]')
+        mask = np.array([False, False])
+
+        rs = s.where(mask, [10, 10])
+        assert_series_equal(rs, expected)
+
+        rs = s.where(mask, 10)
+        assert_series_equal(rs, expected)
+
+        rs = s.where(mask, 10.0)
+        assert_series_equal(rs, expected)
+
+        rs = s.where(mask, [10.0, 10.0])
+        assert_series_equal(rs, expected)
+
+        rs = s.where(mask, [10.0, np.nan])
+        expected = Series([10, None], dtype='datetime64[ns]')
+        assert_series_equal(rs, expected)
+
+    def test_where_timedelta(self):
+        s = Series([1, 2], dtype='timedelta64[ns]')
+        expected = Series([10, 10], dtype='timedelta64[ns]')
+        mask = np.array([False, False])
+
+        rs = s.where(mask, [10, 10])
+        assert_series_equal(rs, expected)
+
+        rs = s.where(mask, 10)
+        assert_series_equal(rs, expected)
+
+        rs = s.where(mask, 10.0)
+        assert_series_equal(rs, expected)
+
+        rs = s.where(mask, [10.0, 10.0])
+        assert_series_equal(rs, expected)
+
+        rs = s.where(mask, [10.0, np.nan])
+        expected = Series([10, None], dtype='timedelta64[ns]')
+        assert_series_equal(rs, expected)
+
     def test_mask(self):
         # compare with tested results in test_where
         s = Series(np.random.randn(5))