BUG: to_dict should return a native datetime object for NumPy backed dataframes

doc/source/whatsnew/v1.2.0.rst

@@ -428,7 +428,7 @@ Numeric
 Conversion
 ^^^^^^^^^^
 
--
+- Bug in :meth:`DataFrame.to_dict` with ``orient='records'`` now returns python native datetime objects for datetimelike columns (:issue:`21256`)
 -
 
 Strings

pandas/core/arrays/sparse/array.py

@@ -22,6 +22,7 @@
     construct_1d_arraylike_from_scalar,
     find_common_type,
     infer_dtype_from_scalar,
+    maybe_box_datetimelike,
 )
 from pandas.core.dtypes.common import (
     is_array_like,
@@ -815,7 +816,7 @@ def _get_val_at(self, loc):
             return self.fill_value
         else:
             val = self.sp_values[sp_loc]
-            val = com.maybe_box_datetimelike(val, self.sp_values.dtype)
+            val = maybe_box_datetimelike(val, self.sp_values.dtype)
             return val
 
     def take(self, indices, allow_fill=False, fill_value=None) -> "SparseArray":

pandas/core/common.py

@@ -6,19 +6,21 @@
 
 from collections import abc, defaultdict
 import contextlib
-from datetime import datetime, timedelta
 from functools import partial
 import inspect
 from typing import Any, Collection, Iterable, Iterator, List, Union, cast
 import warnings
 
 import numpy as np
 
-from pandas._libs import lib, tslibs
+from pandas._libs import lib
 from pandas._typing import AnyArrayLike, Scalar, T
 from pandas.compat.numpy import np_version_under1p18
 
-from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
+from pandas.core.dtypes.cast import (
+    construct_1d_object_array_from_listlike,
+    maybe_box_datetimelike,
+)
 from pandas.core.dtypes.common import (
     is_array_like,
     is_bool_dtype,
@@ -78,21 +80,6 @@ def consensus_name_attr(objs):
     return name
 
 
-def maybe_box_datetimelike(value, dtype=None):
-    # turn a datetime like into a Timestamp/timedelta as needed
-    if dtype == object:
-        # If we dont have datetime64/timedelta64 dtype, we dont want to
-        #  box datetimelike scalars
-        return value
-
-    if isinstance(value, (np.datetime64, datetime)):
-        value = tslibs.Timestamp(value)
-    elif isinstance(value, (np.timedelta64, timedelta)):
-        value = tslibs.Timedelta(value)
-
-    return value
-
-
 def is_bool_indexer(key: Any) -> bool:
     """
     Check whether `key` is a valid boolean indexer.

pandas/core/dtypes/cast.py

@@ -19,7 +19,7 @@
 
 import numpy as np
 
-from pandas._libs import lib, tslib
+from pandas._libs import lib, tslib, tslibs
 from pandas._libs.tslibs import (
     NaT,
     OutOfBoundsDatetime,
@@ -134,6 +134,21 @@ def is_nested_object(obj) -> bool:
     return False
 
 
+def maybe_box_datetimelike(value, dtype=None):
+    # turn a datetime like into a Timestamp/timedelta as needed
+    if dtype == object:
+        # If we dont have datetime64/timedelta64 dtype, we dont want to
+        #  box datetimelike scalars
+        return value
+
+    if isinstance(value, (np.datetime64, datetime)):
+        value = tslibs.Timestamp(value)
+    elif isinstance(value, (np.timedelta64, timedelta)):
+        value = tslibs.Timedelta(value)
+
+    return value
+
+
 def maybe_downcast_to_dtype(result, dtype: Union[str, np.dtype]):
     """
     try to cast to the specified dtype (e.g. convert back to bool/int

pandas/core/frame.py

@@ -83,6 +83,7 @@
     find_common_type,
     infer_dtype_from_scalar,
     invalidate_string_dtypes,
+    maybe_box_datetimelike,
     maybe_cast_to_datetime,
     maybe_casted_values,
     maybe_convert_platform,
@@ -1527,15 +1528,15 @@ def to_dict(self, orient="dict", into=dict):
                     (
                         "data",
                         [
-                            list(map(com.maybe_box_datetimelike, t))
+                            list(map(maybe_box_datetimelike, t))
                             for t in self.itertuples(index=False, name=None)
                         ],
                     ),
                 )
             )
 
         elif orient == "series":
-            return into_c((k, com.maybe_box_datetimelike(v)) for k, v in self.items())
+            return into_c((k, maybe_box_datetimelike(v)) for k, v in self.items())
 
         elif orient == "records":
             columns = self.columns.tolist()
@@ -1544,7 +1545,7 @@ def to_dict(self, orient="dict", into=dict):
                 for row in self.itertuples(index=False, name=None)
             )
             return [
-                into_c((k, com.maybe_box_datetimelike(v)) for k, v in row.items())
+                into_c((k, maybe_box_datetimelike(v)) for k, v in row.items())
                 for row in rows
             ]
 

pandas/core/indexes/interval.py

@@ -19,6 +19,7 @@
 from pandas.core.dtypes.cast import (
     find_common_type,
     infer_dtype_from_scalar,
+    maybe_box_datetimelike,
     maybe_downcast_to_dtype,
 )
 from pandas.core.dtypes.common import (
@@ -1193,8 +1194,8 @@ def interval_range(
     IntervalIndex([[1, 2], [2, 3], [3, 4], [4, 5]],
                   closed='both', dtype='interval[int64]')
     """
-    start = com.maybe_box_datetimelike(start)
-    end = com.maybe_box_datetimelike(end)
+    start = maybe_box_datetimelike(start)
+    end = maybe_box_datetimelike(end)
     endpoint = start if start is not None else end
 
     if freq is None and com.any_none(periods, start, end):

pandas/core/internals/blocks.py

@@ -19,6 +19,7 @@
     find_common_type,
     infer_dtype_from,
     infer_dtype_from_scalar,
+    maybe_box_datetimelike,
     maybe_downcast_numeric,
     maybe_downcast_to_dtype,
     maybe_infer_dtype_type,
@@ -843,7 +844,7 @@ def comp(s: Scalar, mask: np.ndarray, regex: bool = False) -> np.ndarray:
             if isna(s):
                 return ~mask
 
-            s = com.maybe_box_datetimelike(s)
+            s = maybe_box_datetimelike(s)
             return compare_or_regex_search(self.values, s, regex, mask)
 
         # Calculate the mask once, prior to the call of comp

pandas/tests/frame/methods/test_to_dict.py

@@ -257,17 +257,30 @@ def test_to_dict_wide(self):
         assert result == expected
 
     def test_to_dict_orient_dtype(self):
-        # GH#22620
-        # Input Data
-        input_data = {"a": [1, 2, 3], "b": [1.0, 2.0, 3.0], "c": ["X", "Y", "Z"]}
-        df = DataFrame(input_data)
-        # Expected Dtypes
-        expected = {"a": int, "b": float, "c": str}
-        # Extracting dtypes out of to_dict operation
-        for df_dict in df.to_dict("records"):
-            result = {
-                "a": type(df_dict["a"]),
-                "b": type(df_dict["b"]),
-                "c": type(df_dict["c"]),
+        # GH22620 & GH21256
+
+        df = DataFrame(
+            {
+                "bool": [True, True, False],
+                "datetime": [
+                    datetime(2018, 1, 1),
+                    datetime(2019, 2, 2),
+                    datetime(2020, 3, 3),
+                ],
+                "float": [1.0, 2.0, 3.0],
+                "int": [1, 2, 3],
+                "str": ["X", "Y", "Z"],
             }
+        )
+
+        expected = {
+            "int": int,
+            "float": float,
+            "str": str,
+            "datetime": Timestamp,
+            "bool": bool,
+        }
+
+        for df_dict in df.to_dict("records"):
+            result = {col: type(df_dict[col]) for col in list(df.columns)}
             assert result == expected