From af2f84d84d11983da356bc38a2ded3d01dc1f34e Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 1 Nov 2020 22:58:02 +0000 Subject: [PATCH 01/20] BUG/REF: add native arg to maybe_box_datetimelike --- pandas/core/common.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index b860c83f89cbc..c3fc5d653ca2f 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -78,7 +78,7 @@ def consensus_name_attr(objs): return name -def maybe_box_datetimelike(value, dtype=None): +def maybe_box_datetimelike(value, dtype=None, native=False): # turn a datetime like into a Timestamp/timedelta as needed if dtype == object: # If we dont have datetime64/timedelta64 dtype, we dont want to @@ -86,9 +86,11 @@ def maybe_box_datetimelike(value, dtype=None): return value if isinstance(value, (np.datetime64, datetime)): - value = tslibs.Timestamp(value) + ts = tslibs.Timestamp(value) + value = ts.to_pydatetime() if native else ts elif isinstance(value, (np.timedelta64, timedelta)): - value = tslibs.Timedelta(value) + td = tslibs.Timedelta(value) + value = td.to_pydatetime() if native else td return value From 87265bcd658021f4bb241085aaf9294ac4668c80 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 1 Nov 2020 22:59:06 +0000 Subject: [PATCH 02/20] BUG: pass native=True arg to maybe_box_datetimelike --- pandas/core/frame.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 24b89085ac121..050e3099e1b8e 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -1544,7 +1544,10 @@ def to_dict(self, orient="dict", into=dict): for row in self.itertuples(index=False, name=None) ) return [ - into_c((k, com.maybe_box_datetimelike(v)) for k, v in row.items()) + into_c( + (k, com.maybe_box_datetimelike(v, native=True)) + for k, v in row.items() + ) for row in rows ] From 6b8221cf132833ed6a237ad941764ad20e12b622 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 1 Nov 2020 22:59:49 +0000 Subject: [PATCH 03/20] TST: add datetime column to test --- pandas/tests/frame/methods/test_to_dict.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pandas/tests/frame/methods/test_to_dict.py b/pandas/tests/frame/methods/test_to_dict.py index f1656b46cf356..a5843a97c66f8 100644 --- a/pandas/tests/frame/methods/test_to_dict.py +++ b/pandas/tests/frame/methods/test_to_dict.py @@ -259,15 +259,21 @@ def test_to_dict_wide(self): def test_to_dict_orient_dtype(self): # GH#22620 # Input Data - input_data = {"a": [1, 2, 3], "b": [1.0, 2.0, 3.0], "c": ["X", "Y", "Z"]} + input_data = { + "a": [1, 2, 3], + "b": [1.0, 2.0, 3.0], + "c": ["X", "Y", "Z"], + "d": [datetime(2018, 1, 1), datetime(2019, 2, 2), datetime(2020, 3, 3)], + } df = DataFrame(input_data) # Expected Dtypes - expected = {"a": int, "b": float, "c": str} + expected = {"a": int, "b": float, "c": str, "d": datetime} # Extracting dtypes out of to_dict operation for df_dict in df.to_dict("records"): result = { "a": type(df_dict["a"]), "b": type(df_dict["b"]), "c": type(df_dict["c"]), + "d": type(df_dict["d"]), } assert result == expected From a28bfad8cfb7208f7f0d805a2fa8d2509db78c6d Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 1 Nov 2020 23:05:41 +0000 Subject: [PATCH 04/20] TST/CLN: test comments --- pandas/tests/frame/methods/test_to_dict.py | 23 +++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pandas/tests/frame/methods/test_to_dict.py b/pandas/tests/frame/methods/test_to_dict.py index a5843a97c66f8..6d6fe51cb6e52 100644 --- a/pandas/tests/frame/methods/test_to_dict.py +++ b/pandas/tests/frame/methods/test_to_dict.py @@ -257,18 +257,19 @@ def test_to_dict_wide(self): assert result == expected def test_to_dict_orient_dtype(self): - # GH#22620 - # Input Data - input_data = { - "a": [1, 2, 3], - "b": [1.0, 2.0, 3.0], - "c": ["X", "Y", "Z"], - "d": [datetime(2018, 1, 1), datetime(2019, 2, 2), datetime(2020, 3, 3)], - } - df = DataFrame(input_data) - # Expected Dtypes + # GH22620 & GH21256 + + df = DataFrame( + { + "a": [1, 2, 3], + "b": [1.0, 2.0, 3.0], + "c": ["X", "Y", "Z"], + "d": [datetime(2018, 1, 1), datetime(2019, 2, 2), datetime(2020, 3, 3)], + } + ) + expected = {"a": int, "b": float, "c": str, "d": datetime} - # Extracting dtypes out of to_dict operation + for df_dict in df.to_dict("records"): result = { "a": type(df_dict["a"]), From 8821a8d1965c8fad35a02dd9202103fc2ee75106 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 1 Nov 2020 23:14:16 +0000 Subject: [PATCH 05/20] DOC: whatsnew --- doc/source/whatsnew/v1.2.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 45a95f6aeb2f6..c490ee88bda8e 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -428,7 +428,7 @@ Numeric Conversion ^^^^^^^^^^ -- +- Bug in :meth:`DataFrame.to_dict` with ``orient='records'`` now returns python native datetime objects for datetimelike columns (:issue:`21256`) - Strings From 717a6e397907b47eeaa895370ace4ac4d2f07cd9 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 1 Nov 2020 23:24:42 +0000 Subject: [PATCH 06/20] CLN/TST: add bool test case & DRY the test --- pandas/tests/frame/methods/test_to_dict.py | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/pandas/tests/frame/methods/test_to_dict.py b/pandas/tests/frame/methods/test_to_dict.py index 6d6fe51cb6e52..5d22b8407f9e8 100644 --- a/pandas/tests/frame/methods/test_to_dict.py +++ b/pandas/tests/frame/methods/test_to_dict.py @@ -261,20 +261,26 @@ def test_to_dict_orient_dtype(self): df = DataFrame( { - "a": [1, 2, 3], - "b": [1.0, 2.0, 3.0], - "c": ["X", "Y", "Z"], - "d": [datetime(2018, 1, 1), datetime(2019, 2, 2), datetime(2020, 3, 3)], + "bool": [True, True, False], + "datetime": [ + datetime(2018, 1, 1), + datetime(2019, 2, 2), + datetime(2020, 3, 3), + ], + "float": [1.0, 2.0, 3.0], + "int": [1, 2, 3], + "str": ["X", "Y", "Z"], } ) - expected = {"a": int, "b": float, "c": str, "d": datetime} + expected = { + "int": int, + "float": float, + "str": str, + "datetime": datetime, + "bool": bool, + } for df_dict in df.to_dict("records"): - result = { - "a": type(df_dict["a"]), - "b": type(df_dict["b"]), - "c": type(df_dict["c"]), - "d": type(df_dict["d"]), - } + result = {col: type(df_dict[col]) for col in list(df.columns)} assert result == expected From ad7819063b2a994ef0aa0354e802d853d3ed0dc0 Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Tue, 3 Nov 2020 06:20:23 +0000 Subject: [PATCH 07/20] REF (feedback): move maybe_box_datetimelike common.py -> dtypes/cast.py --- pandas/core/arrays/sparse/array.py | 3 ++- pandas/core/common.py | 25 +++++-------------------- pandas/core/dtypes/cast.py | 15 +++++++++++++++ pandas/core/frame.py | 10 ++++------ pandas/core/indexes/interval.py | 5 +++-- pandas/core/internals/blocks.py | 3 ++- 6 files changed, 31 insertions(+), 30 deletions(-) diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py index 4346e02069667..7e5b7935dc436 100644 --- a/pandas/core/arrays/sparse/array.py +++ b/pandas/core/arrays/sparse/array.py @@ -22,6 +22,7 @@ construct_1d_arraylike_from_scalar, find_common_type, infer_dtype_from_scalar, + maybe_box_datetimelike, ) from pandas.core.dtypes.common import ( is_array_like, @@ -815,7 +816,7 @@ def _get_val_at(self, loc): return self.fill_value else: val = self.sp_values[sp_loc] - val = com.maybe_box_datetimelike(val, self.sp_values.dtype) + val = maybe_box_datetimelike(val, self.sp_values.dtype) return val def take(self, indices, allow_fill=False, fill_value=None) -> "SparseArray": diff --git a/pandas/core/common.py b/pandas/core/common.py index c3fc5d653ca2f..ccbc052d9356b 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -6,7 +6,6 @@ from collections import abc, defaultdict import contextlib -from datetime import datetime, timedelta from functools import partial import inspect from typing import Any, Collection, Iterable, Iterator, List, Union, cast @@ -14,11 +13,14 @@ import numpy as np -from pandas._libs import lib, tslibs +from pandas._libs import lib from pandas._typing import AnyArrayLike, Scalar, T from pandas.compat.numpy import np_version_under1p18 -from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike +from pandas.core.dtypes.cast import ( + construct_1d_object_array_from_listlike, + maybe_box_datetimelike, +) from pandas.core.dtypes.common import ( is_array_like, is_bool_dtype, @@ -78,23 +80,6 @@ def consensus_name_attr(objs): return name -def maybe_box_datetimelike(value, dtype=None, native=False): - # turn a datetime like into a Timestamp/timedelta as needed - if dtype == object: - # If we dont have datetime64/timedelta64 dtype, we dont want to - # box datetimelike scalars - return value - - if isinstance(value, (np.datetime64, datetime)): - ts = tslibs.Timestamp(value) - value = ts.to_pydatetime() if native else ts - elif isinstance(value, (np.timedelta64, timedelta)): - td = tslibs.Timedelta(value) - value = td.to_pydatetime() if native else td - - return value - - def is_bool_indexer(key: Any) -> bool: """ Check whether `key` is a valid boolean indexer. diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 692da8f8e021e..c4712cf7f39bb 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -134,6 +134,21 @@ def is_nested_object(obj) -> bool: return False +def maybe_box_datetimelike(value, dtype=None): + # turn a datetime like into a Timestamp/timedelta as needed + if dtype == object: + # If we dont have datetime64/timedelta64 dtype, we dont want to + # box datetimelike scalars + return value + + if isinstance(value, (np.datetime64, datetime)): + value = tslib.Timestamp(value) + elif isinstance(value, (np.timedelta64, timedelta)): + value = tslib.Timedelta(value) + + return value + + def maybe_downcast_to_dtype(result, dtype: Union[str, np.dtype]): """ try to cast to the specified dtype (e.g. convert back to bool/int diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 050e3099e1b8e..26808bf208608 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -83,6 +83,7 @@ find_common_type, infer_dtype_from_scalar, invalidate_string_dtypes, + maybe_box_datetimelike, maybe_cast_to_datetime, maybe_casted_values, maybe_convert_platform, @@ -1527,7 +1528,7 @@ def to_dict(self, orient="dict", into=dict): ( "data", [ - list(map(com.maybe_box_datetimelike, t)) + list(map(maybe_box_datetimelike, t)) for t in self.itertuples(index=False, name=None) ], ), @@ -1535,7 +1536,7 @@ def to_dict(self, orient="dict", into=dict): ) elif orient == "series": - return into_c((k, com.maybe_box_datetimelike(v)) for k, v in self.items()) + return into_c((k, maybe_box_datetimelike(v)) for k, v in self.items()) elif orient == "records": columns = self.columns.tolist() @@ -1544,10 +1545,7 @@ def to_dict(self, orient="dict", into=dict): for row in self.itertuples(index=False, name=None) ) return [ - into_c( - (k, com.maybe_box_datetimelike(v, native=True)) - for k, v in row.items() - ) + into_c((k, maybe_box_datetimelike(v)) for k, v in row.items()) for row in rows ] diff --git a/pandas/core/indexes/interval.py b/pandas/core/indexes/interval.py index 1bd71f00b534d..1a654e6ab8cf9 100644 --- a/pandas/core/indexes/interval.py +++ b/pandas/core/indexes/interval.py @@ -36,6 +36,7 @@ is_number, is_object_dtype, is_scalar, + maybe_box_datetimelike, ) from pandas.core.algorithms import take_1d @@ -1206,8 +1207,8 @@ def interval_range( IntervalIndex([[1, 2], [2, 3], [3, 4], [4, 5]], closed='both', dtype='interval[int64]') """ - start = com.maybe_box_datetimelike(start) - end = com.maybe_box_datetimelike(end) + start = maybe_box_datetimelike(start) + end = maybe_box_datetimelike(end) endpoint = start if start is not None else end if freq is None and com.any_none(periods, start, end): diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 24b00199611bf..c853f0f83b8f9 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -19,6 +19,7 @@ find_common_type, infer_dtype_from, infer_dtype_from_scalar, + maybe_box_datetimelike, maybe_downcast_numeric, maybe_downcast_to_dtype, maybe_infer_dtype_type, @@ -843,7 +844,7 @@ def comp(s: Scalar, mask: np.ndarray, regex: bool = False) -> np.ndarray: if isna(s): return ~mask - s = com.maybe_box_datetimelike(s) + s = maybe_box_datetimelike(s) return compare_or_regex_search(self.values, s, regex, mask) # Calculate the mask once, prior to the call of comp From c93fca9b2d47edef84255d5efdd1b62ab97fffd9 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 2 Nov 2020 19:15:28 +0000 Subject: [PATCH 08/20] feedback: remove native arg/accept Timestamp return type --- pandas/tests/frame/methods/test_to_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/methods/test_to_dict.py b/pandas/tests/frame/methods/test_to_dict.py index 5d22b8407f9e8..f8feef7a95eab 100644 --- a/pandas/tests/frame/methods/test_to_dict.py +++ b/pandas/tests/frame/methods/test_to_dict.py @@ -277,7 +277,7 @@ def test_to_dict_orient_dtype(self): "int": int, "float": float, "str": str, - "datetime": datetime, + "datetime": Timestamp, "bool": bool, } From de8665241d8113405d5a05d4a4d1162f946cb3a2 Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Tue, 3 Nov 2020 16:05:41 +0000 Subject: [PATCH 09/20] BUG: fix import in pd.core.indexes.interval.py --- pandas/core/indexes/interval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/interval.py b/pandas/core/indexes/interval.py index bf2d82dfe750b..c700acc24f411 100644 --- a/pandas/core/indexes/interval.py +++ b/pandas/core/indexes/interval.py @@ -19,6 +19,7 @@ from pandas.core.dtypes.cast import ( find_common_type, infer_dtype_from_scalar, + maybe_box_datetimelike, maybe_downcast_to_dtype, ) from pandas.core.dtypes.common import ( @@ -36,7 +37,6 @@ is_number, is_object_dtype, is_scalar, - maybe_box_datetimelike, ) from pandas.core.algorithms import take_1d From b17a0db26c282097b2934aeadd187fe1446a9cb1 Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Tue, 3 Nov 2020 16:22:54 +0000 Subject: [PATCH 10/20] BUG: fix import in pd.core.dtypes.cast.py --- pandas/core/dtypes/cast.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index c4712cf7f39bb..5a487340930f6 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -19,7 +19,7 @@ import numpy as np -from pandas._libs import lib, tslib +from pandas._libs import lib, tslibs from pandas._libs.tslibs import ( NaT, OutOfBoundsDatetime, @@ -142,9 +142,9 @@ def maybe_box_datetimelike(value, dtype=None): return value if isinstance(value, (np.datetime64, datetime)): - value = tslib.Timestamp(value) + value = tslibs.Timestamp(value) elif isinstance(value, (np.timedelta64, timedelta)): - value = tslib.Timedelta(value) + value = tslibs.Timedelta(value) return value @@ -1352,7 +1352,7 @@ def try_datetime(v): # safe coerce to datetime64 try: # GH19671 - v = tslib.array_to_datetime(v, require_iso8601=True, errors="raise")[0] + v = tslibs.array_to_datetime(v, require_iso8601=True, errors="raise")[0] except ValueError: # we might have a sequence of the same-datetimes with tz's From 1057c8a09beb26eb970ad76e90356a3035a7a3ac Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Tue, 3 Nov 2020 20:01:01 +0000 Subject: [PATCH 11/20] BUG: revert accidental change tslib->tslibs --- pandas/core/dtypes/cast.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 5a487340930f6..a7857ca0cbf86 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -19,7 +19,7 @@ import numpy as np -from pandas._libs import lib, tslibs +from pandas._libs import lib, tslib, tslibs from pandas._libs.tslibs import ( NaT, OutOfBoundsDatetime, @@ -1352,7 +1352,7 @@ def try_datetime(v): # safe coerce to datetime64 try: # GH19671 - v = tslibs.array_to_datetime(v, require_iso8601=True, errors="raise")[0] + v = tslib.array_to_datetime(v, require_iso8601=True, errors="raise")[0] except ValueError: # we might have a sequence of the same-datetimes with tz's From 10119a7dc9a930035836aabaefdf7cc103cd7cc4 Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 17:25:58 +0000 Subject: [PATCH 12/20] REF (feedback): move dict_compat to core/dtypes/cast.py --- pandas/core/common.py | 22 +--------------------- pandas/core/dtypes/cast.py | 17 +++++++++++++++++ pandas/core/internals/construction.py | 3 ++- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index ccbc052d9356b..9b6133d2f7627 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -17,10 +17,7 @@ from pandas._typing import AnyArrayLike, Scalar, T from pandas.compat.numpy import np_version_under1p18 -from pandas.core.dtypes.cast import ( - construct_1d_object_array_from_listlike, - maybe_box_datetimelike, -) +from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike from pandas.core.dtypes.common import ( is_array_like, is_bool_dtype, @@ -334,23 +331,6 @@ def apply_if_callable(maybe_callable, obj, **kwargs): return maybe_callable -def dict_compat(d): - """ - Helper function to convert datetimelike-keyed dicts - to Timestamp-keyed dict. - - Parameters - ---------- - d: dict like object - - Returns - ------- - dict - - """ - return {maybe_box_datetimelike(key): value for key, value in d.items()} - - def standardize_mapping(into): """ Helper function to standardize a supplied mapping. diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index a7857ca0cbf86..eda8000597012 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -806,6 +806,23 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False) -> Tuple[DtypeObj, return dtype, val +def dict_compat(d): + """ + Helper function to convert datetimelike-keyed dicts + to Timestamp-keyed dict. + + Parameters + ---------- + d: dict like object + + Returns + ------- + dict + + """ + return {maybe_box_datetimelike(key): value for key, value in d.items()} + + def infer_dtype_from_array( arr, pandas_dtype: bool = False ) -> Tuple[DtypeObj, ArrayLike]: diff --git a/pandas/core/internals/construction.py b/pandas/core/internals/construction.py index bb8283604abb0..bcafa2c2fdca7 100644 --- a/pandas/core/internals/construction.py +++ b/pandas/core/internals/construction.py @@ -14,6 +14,7 @@ from pandas.core.dtypes.cast import ( construct_1d_arraylike_from_scalar, construct_1d_ndarray_preserving_na, + dict_compat, maybe_cast_to_datetime, maybe_convert_platform, maybe_infer_to_datetimelike, @@ -346,7 +347,7 @@ def _homogenize(data, index, dtype: Optional[DtypeObj]): oindex = index.astype("O") if isinstance(index, (ABCDatetimeIndex, ABCTimedeltaIndex)): - val = com.dict_compat(val) + val = dict_compat(val) else: val = dict(val) val = lib.fast_multiget(val, oindex._values, default=np.nan) From 44d82c490cf566a84f41b9d8637e1ccaf631447d Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 17:32:04 +0000 Subject: [PATCH 13/20] REF/TST: move dict_compat tests to dtypes/cast/test_dict_compat.py --- pandas/tests/dtypes/cast/test_dict_compat.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 pandas/tests/dtypes/cast/test_dict_compat.py diff --git a/pandas/tests/dtypes/cast/test_dict_compat.py b/pandas/tests/dtypes/cast/test_dict_compat.py new file mode 100644 index 0000000000000..13dc82d779f95 --- /dev/null +++ b/pandas/tests/dtypes/cast/test_dict_compat.py @@ -0,0 +1,14 @@ +import numpy as np + +from pandas.core.dtypes.cast import dict_compat + +from pandas import Timestamp + + +def test_dict_compat(): + data_datetime64 = {np.datetime64("1990-03-15"): 1, np.datetime64("2015-03-15"): 2} + data_unchanged = {1: 2, 3: 4, 5: 6} + expected = {Timestamp("1990-3-15"): 1, Timestamp("2015-03-15"): 2} + assert dict_compat(data_datetime64) == expected + assert dict_compat(expected) == expected + assert dict_compat(data_unchanged) == data_unchanged From 5d8d3748972adaf4dabe05c8065d928601b793c1 Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 17:35:38 +0000 Subject: [PATCH 14/20] DOC: improve dict_compat doctring --- pandas/core/dtypes/cast.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index ec63c50fca25e..00c362025e1de 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -808,12 +808,11 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False) -> Tuple[DtypeObj, def dict_compat(d): """ - Helper function to convert datetimelike-keyed dicts - to Timestamp-keyed dict. + Convert datetimelike-keyed dicts to a Timestamp-keyed dict. Parameters ---------- - d: dict like object + d: dict-like object Returns ------- From b9b8e3103306b86189cbc38efd619f2fc18c7e60 Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 17:44:33 +0000 Subject: [PATCH 15/20] DOC: add maybe_box_datetimelike docstring --- pandas/core/dtypes/cast.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 00c362025e1de..196dbe7c348bc 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -135,6 +135,20 @@ def is_nested_object(obj) -> bool: def maybe_box_datetimelike(value, dtype=None): + """ + Cast scalar to Timestamp or Timedelta if scalar is datetime-like + and dtype is not object. + + Parameters + ---------- + value : scalar + dtype : Dtype, optional + + Returns + ------- + scalar + """ + # turn a datetime like into a Timestamp/timedelta as needed if dtype == object: # If we dont have datetime64/timedelta64 dtype, we dont want to From f1667f602f2b625ba9f9478599fec374e83fee44 Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 17:46:42 +0000 Subject: [PATCH 16/20] CLN: consolidate return in maybe_box_datetimelike --- pandas/core/dtypes/cast.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 196dbe7c348bc..257dbc6be0623 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -148,14 +148,9 @@ def maybe_box_datetimelike(value, dtype=None): ------- scalar """ - - # turn a datetime like into a Timestamp/timedelta as needed if dtype == object: - # If we dont have datetime64/timedelta64 dtype, we dont want to - # box datetimelike scalars - return value - - if isinstance(value, (np.datetime64, datetime)): + pass + elif isinstance(value, (np.datetime64, datetime)): value = tslibs.Timestamp(value) elif isinstance(value, (np.timedelta64, timedelta)): value = tslibs.Timedelta(value) From 5b984b83003235f3697b6df9cbf531cfba6e4325 Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 19:17:48 +0000 Subject: [PATCH 17/20] TYP: maybe_box_datetimelike --- pandas/core/dtypes/cast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 257dbc6be0623..0a18df537b675 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -134,7 +134,7 @@ def is_nested_object(obj) -> bool: return False -def maybe_box_datetimelike(value, dtype=None): +def maybe_box_datetimelike(value: Scalar, dtype: Optional[Dtype] = None): """ Cast scalar to Timestamp or Timedelta if scalar is datetime-like and dtype is not object. From 017fa587b9a0fa54c5ca00c2da0a408520020d8f Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 19:32:31 +0000 Subject: [PATCH 18/20] TYP: maybe_box_datetimelike (return type) --- pandas/core/dtypes/cast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 0a18df537b675..ba0bc03b7f057 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -134,7 +134,7 @@ def is_nested_object(obj) -> bool: return False -def maybe_box_datetimelike(value: Scalar, dtype: Optional[Dtype] = None): +def maybe_box_datetimelike(value: Scalar, dtype: Optional[Dtype] = None) -> Scalar: """ Cast scalar to Timestamp or Timedelta if scalar is datetime-like and dtype is not object. From c76c09c37d24203655d88dcd0d20fbdbff708f2d Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 19:49:39 +0000 Subject: [PATCH 19/20] TYP: dict_compat --- pandas/core/dtypes/cast.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index ba0bc03b7f057..9758eae60c262 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -7,6 +7,7 @@ from typing import ( TYPE_CHECKING, Any, + Dict, List, Optional, Sequence, @@ -815,7 +816,7 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False) -> Tuple[DtypeObj, return dtype, val -def dict_compat(d): +def dict_compat(d: Dict[Scalar, Scalar]) -> Dict[Scalar, Scalar]: """ Convert datetimelike-keyed dicts to a Timestamp-keyed dict. From 2697b850a5b30822a535d541be0089c292b1be9c Mon Sep 17 00:00:00 2001 From: Andrew Wieteska Date: Wed, 4 Nov 2020 20:11:11 +0000 Subject: [PATCH 20/20] TST: remove dict_compat test from test_common.py --- pandas/tests/test_common.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pandas/tests/test_common.py b/pandas/tests/test_common.py index 366a1970f6f64..81d866ba63bc0 100644 --- a/pandas/tests/test_common.py +++ b/pandas/tests/test_common.py @@ -9,7 +9,7 @@ from pandas.compat.numpy import np_version_under1p17 import pandas as pd -from pandas import Series, Timestamp +from pandas import Series import pandas._testing as tm from pandas.core import ops import pandas.core.common as com @@ -109,15 +109,6 @@ def test_maybe_match_name(left, right, expected): assert ops.common._maybe_match_name(left, right) == expected -def test_dict_compat(): - data_datetime64 = {np.datetime64("1990-03-15"): 1, np.datetime64("2015-03-15"): 2} - data_unchanged = {1: 2, 3: 4, 5: 6} - expected = {Timestamp("1990-3-15"): 1, Timestamp("2015-03-15"): 2} - assert com.dict_compat(data_datetime64) == expected - assert com.dict_compat(expected) == expected - assert com.dict_compat(data_unchanged) == data_unchanged - - def test_standardize_mapping(): # No uninitialized defaultdicts msg = r"to_dict\(\) only accepts initialized defaultdicts"