From ec91b649c269dd7d8e6f9db0b6b3938f0d1fb129 Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 11 Feb 2021 13:40:24 -0800 Subject: [PATCH 1/2] BUG: casting dt64/td64 in DataFrame.reindex --- pandas/core/algorithms.py | 3 ++ pandas/tests/frame/methods/test_reindex.py | 35 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index 98069e73dfd0c..a55f857dc2357 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -1388,6 +1388,9 @@ def wrapper(arr, indexer, out, fill_value=np.nan): def _convert_wrapper(f, conv_dtype): def wrapper(arr, indexer, out, fill_value=np.nan): + if conv_dtype == object: + # GH#39755 avoid casting dt64/td64 to integers + arr = ensure_wrapped_if_datetimelike(arr) arr = arr.astype(conv_dtype) f(arr, indexer, out, fill_value=fill_value) diff --git a/pandas/tests/frame/methods/test_reindex.py b/pandas/tests/frame/methods/test_reindex.py index 9116b1ff5ad65..e4e2656f4337c 100644 --- a/pandas/tests/frame/methods/test_reindex.py +++ b/pandas/tests/frame/methods/test_reindex.py @@ -933,3 +933,38 @@ def test_reindex_empty(self, src_idx, cat_idx): result = df.reindex(columns=cat_idx) expected = DataFrame(index=["K"], columns=cat_idx, dtype="f8") tm.assert_frame_equal(result, expected) + + @pytest.mark.parametrize("dtype", ["m8[ns]", "M8[ns]"]) + def test_reindex_datetimelike_to_object(self, dtype): + # GH#39755 dont cast dt64/td64 to ints + mi = MultiIndex.from_product([list("ABCDE"), range(2)]) + + dti = date_range("2016-01-01", periods=10) + fv = np.timedelta64("NaT", "ns") + if dtype == "m8[ns]": + dti = dti - dti[0] + fv = np.datetime64("NaT", "ns") + + ser = Series(dti, index=mi) + ser[::3] = pd.NaT + + df = ser.unstack() + + index = df.index.append(Index([1])) + columns = df.columns.append(Index(["foo"])) + + res = df.reindex(index=index, columns=columns, fill_value=fv) + + expected = DataFrame( + { + 0: df[0].tolist() + [fv], + 1: df[1].tolist() + [fv], + "foo": np.array(["NaT"] * 6, dtype=fv.dtype), + }, + index=index, + ) + assert (res.dtypes[[0, 1]] == object).all() + assert res.iloc[0, 0] is pd.NaT + assert res.iloc[-1, 0] is fv + assert res.iloc[-1, 1] is fv + tm.assert_frame_equal(res, expected) From 5b1249d41fd25ca5437755e2086926015a0601da Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 11 Feb 2021 13:42:07 -0800 Subject: [PATCH 2/2] whatsnew --- doc/source/whatsnew/v1.3.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 09e1853429d9f..0d28177fa30e4 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -335,6 +335,7 @@ Indexing - Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` when setting multiple values to duplicate columns (:issue:`15695`) - Bug in :meth:`DataFrame.loc`, :meth:`Series.loc`, :meth:`DataFrame.__getitem__` and :meth:`Series.__getitem__` returning incorrect elements for non-monotonic :class:`DatetimeIndex` for string slices (:issue:`33146`) - Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` with timezone aware indexes raising ``TypeError`` for ``method="ffill"`` and ``method="bfill"`` and specified ``tolerance`` (:issue:`38566`) +- Bug in :meth:`DataFrame.reindex` with ``datetime64[ns]`` or ``timedelta64[ns]`` incorrectly casting to integers when the ``fill_value`` requires casting to object dtype (:issue:`39755`) - Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` with empty :class:`DataFrame` and specified columns for string indexer and non empty :class:`DataFrame` to set (:issue:`38831`) - Bug in :meth:`DataFrame.loc.__setitem__` raising ValueError when expanding unique column for :class:`DataFrame` with duplicate columns (:issue:`38521`) - Bug in :meth:`DataFrame.iloc.__setitem__` and :meth:`DataFrame.loc.__setitem__` with mixed dtypes when setting with a dictionary value (:issue:`38335`)