DEPR: casting date to dt64 in maybe_promote (#39767)

Author: jbrockmendel

doc/source/whatsnew/v1.3.0.rst

@@ -239,7 +239,7 @@ Deprecations
 - Deprecated :attr:`Rolling.is_datetimelike` (:issue:`38963`)
 - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`)
 - Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`)
--
+- Deprecated casting ``datetime.date`` objects to ``datetime64`` when used as ``fill_value`` in :meth:`DataFrame.unstack`, :meth:`DataFrame.shift`, :meth:`Series.shift`, and :meth:`DataFrame.reindex`, pass ``pd.Timestamp(dateobj)`` instead (:issue:`39767`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/dtypes/cast.py

@@ -5,7 +5,7 @@
 from __future__ import annotations
 
 from contextlib import suppress
-from datetime import datetime, timedelta
+from datetime import date, datetime, timedelta
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -549,16 +549,46 @@ def maybe_promote(dtype: np.dtype, fill_value=np.nan):
 
     # returns tuple of (dtype, fill_value)
     if issubclass(dtype.type, np.datetime64):
-        if isinstance(fill_value, datetime) and fill_value.tzinfo is not None:
-            # Trying to insert tzaware into tznaive, have to cast to object
-            dtype = np.dtype(np.object_)
-        elif is_integer(fill_value) or is_float(fill_value):
-            dtype = np.dtype(np.object_)
-        else:
+        inferred, fv = infer_dtype_from_scalar(fill_value, pandas_dtype=True)
+        if inferred == dtype:
+            return dtype, fv
+
+        # TODO(2.0): once this deprecation is enforced, this whole case
+        # becomes equivalent to:
+        #  dta = DatetimeArray._from_sequence([], dtype="M8[ns]")
+        #  try:
+        #      fv = dta._validate_setitem_value(fill_value)
+        #      return dta.dtype, fv
+        #  except (ValueError, TypeError):
+        #      return np.dtype(object), fill_value
+        if isinstance(fill_value, date) and not isinstance(fill_value, datetime):
+            # deprecate casting of date object to match infer_dtype_from_scalar
+            #  and DatetimeArray._validate_setitem_value
             try:
-                fill_value = Timestamp(fill_value).to_datetime64()
-            except (TypeError, ValueError):
-                dtype = np.dtype(np.object_)
+                fv = Timestamp(fill_value).to_datetime64()
+            except OutOfBoundsDatetime:
+                pass
+            else:
+                warnings.warn(
+                    "Using a `date` object for fill_value with `datetime64[ns]` "
+                    "dtype is deprecated. In a future version, this will be cast "
+                    "to object dtype. Pass `fill_value=Timestamp(date_obj)` instead.",
+                    FutureWarning,
+                    stacklevel=7,
+                )
+                return dtype, fv
+        elif isinstance(fill_value, str):
+            try:
+                # explicitly wrap in str to convert np.str_
+                fv = Timestamp(str(fill_value))
+            except (ValueError, TypeError):
+                pass
+            else:
+                if fv.tz is None:
+                    return dtype, fv.asm8
+
+        return np.dtype(object), fill_value
+
     elif issubclass(dtype.type, np.timedelta64):
         if (
             is_integer(fill_value)
@@ -723,13 +753,13 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False) -> Tuple[DtypeObj,
 
         if val is NaT or val.tz is None:
             dtype = np.dtype("M8[ns]")
+            val = val.to_datetime64()
         else:
             if pandas_dtype:
                 dtype = DatetimeTZDtype(unit="ns", tz=val.tz)
             else:
                 # return datetimetz as object
                 return np.dtype(object), val
-        val = val.value
 
     elif isinstance(val, (np.timedelta64, timedelta)):
         try:

pandas/core/indexes/interval.py

@@ -533,6 +533,10 @@ def _maybe_convert_i8(self, key):
             key_dtype, key_i8 = infer_dtype_from_scalar(key, pandas_dtype=True)
             if lib.is_period(key):
                 key_i8 = key.ordinal
+            elif isinstance(key_i8, Timestamp):
+                key_i8 = key_i8.value
+            elif isinstance(key_i8, (np.datetime64, np.timedelta64)):
+                key_i8 = key_i8.view("i8")
         else:
             # DatetimeIndex/TimedeltaIndex
             key_dtype, key_i8 = key.dtype, Index(key.asi8)

pandas/tests/dtypes/cast/test_infer_dtype.py

@@ -105,13 +105,11 @@ def test_infer_from_scalar_tz(tz, pandas_dtype):
 
     if pandas_dtype:
         exp_dtype = f"datetime64[ns, {tz}]"
-        exp_val = dt.value
     else:
         exp_dtype = np.object_
-        exp_val = dt
 
     assert dtype == exp_dtype
-    assert val == exp_val
+    assert val == dt
 
 
 @pytest.mark.parametrize(

pandas/tests/dtypes/cast/test_promote.py

@@ -24,6 +24,7 @@
 from pandas.core.dtypes.missing import isna
 
 import pandas as pd
+import pandas._testing as tm
 
 
 @pytest.fixture(
@@ -403,7 +404,13 @@ def test_maybe_promote_any_with_datetime64(
         expected_dtype = np.dtype(object)
         exp_val_for_scalar = fill_value
 
-    _check_promote(dtype, fill_value, expected_dtype, exp_val_for_scalar)
+    warn = None
+    if type(fill_value) is datetime.date and dtype.kind == "M":
+        # Casting date to dt64 is deprecated
+        warn = FutureWarning
+
+    with tm.assert_produces_warning(warn, check_stacklevel=False):
+        _check_promote(dtype, fill_value, expected_dtype, exp_val_for_scalar)
 
 
 @pytest.mark.parametrize(

pandas/tests/frame/methods/test_reindex.py

@@ -62,6 +62,28 @@ class TestDataFrameSelectReindex:
     # These are specific reindex-based tests; other indexing tests should go in
     # test_indexing
 
+    def test_reindex_date_fill_value(self):
+        # passing date to dt64 is deprecated
+        arr = date_range("2016-01-01", periods=6).values.reshape(3, 2)
+        df = DataFrame(arr, columns=["A", "B"], index=range(3))
+
+        ts = df.iloc[0, 0]
+        fv = ts.date()
+
+        with tm.assert_produces_warning(FutureWarning):
+            res = df.reindex(index=range(4), columns=["A", "B", "C"], fill_value=fv)
+
+        expected = DataFrame(
+            {"A": df["A"].tolist() + [ts], "B": df["B"].tolist() + [ts], "C": [ts] * 4}
+        )
+        tm.assert_frame_equal(res, expected)
+
+        # same with a datetime-castable str
+        res = df.reindex(
+            index=range(4), columns=["A", "B", "C"], fill_value="2016-01-01"
+        )
+        tm.assert_frame_equal(res, expected)
+
     def test_reindex_with_multi_index(self):
         # https://github.com/pandas-dev/pandas/issues/29896
         # tests for reindexing a multi-indexed DataFrame with a new MultiIndex