From a8eba60bdca95d386fefde5a166c894a1f38d423 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sat, 10 Oct 2020 19:34:22 +0000 Subject: [PATCH 01/10] CLN: rewrite try/except/pass using contextlib.suppress --- pandas/core/dtypes/cast.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index a7379376c2f78..ac0a2a82b4fdc 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -2,6 +2,7 @@ Routines for casting. """ +from contextlib import suppress from datetime import date, datetime, timedelta from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type @@ -176,12 +177,10 @@ def maybe_downcast_to_dtype(result, dtype): # TODO(DatetimeArray): merge with previous elif from pandas.core.arrays import PeriodArray - try: - return PeriodArray(result, freq=dtype.freq) - except TypeError: + with suppress(TypeError): # e.g. TypeError: int() argument must be a string, a # bytes-like object or a number, not 'Period - pass + return PeriodArray(result, freq=dtype.freq) return result From 058bf6f5b8e5fd08b6d420160bd76a660c8e2e82 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sat, 10 Oct 2020 20:30:58 +0000 Subject: [PATCH 02/10] use is_period_dtype to check type --- pandas/core/dtypes/cast.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index ac0a2a82b4fdc..3b18e41569d8a 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -12,7 +12,6 @@ from pandas._libs.tslibs import ( NaT, OutOfBoundsDatetime, - Period, Timedelta, Timestamp, iNaT, @@ -50,6 +49,7 @@ is_integer_dtype, is_numeric_dtype, is_object_dtype, + is_period_dtype, is_scalar, is_sparse, is_string_dtype, @@ -173,7 +173,7 @@ def maybe_downcast_to_dtype(result, dtype): else: result = result.astype(dtype) - elif dtype.type is Period: + elif is_period_dtype(dtype): # TODO(DatetimeArray): merge with previous elif from pandas.core.arrays import PeriodArray From 9eb0b67f9702d6d275397d0eb452df87f0686436 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sat, 10 Oct 2020 22:07:21 +0000 Subject: [PATCH 03/10] clean up if condition --- pandas/core/dtypes/cast.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 3b18e41569d8a..7481e0f710448 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -40,6 +40,7 @@ is_datetime64_dtype, is_datetime64_ns_dtype, is_datetime64tz_dtype, + is_datetime_or_timedelta_any_dtype, is_datetime_or_timedelta_dtype, is_dtype_equal, is_extension_array_dtype, @@ -160,9 +161,9 @@ def maybe_downcast_to_dtype(result, dtype): return converted # a datetimelike - # GH12821, iNaT is casted to float - if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: - if hasattr(dtype, "tz"): + # GH12821, iNaT is cast to float + if is_datetime_or_timedelta_any_dtype(dtype) and result.dtype.kind in ["i", "f"]: + if not is_datetime_or_timedelta_dtype(dtype): # not a numpy dtype if dtype.tz: # convert to datetime and change timezone From c2ad3d95d13e377a5ed9e489eac69516f744f20c Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sat, 10 Oct 2020 22:21:07 +0000 Subject: [PATCH 04/10] add is_datetime_or_timedelta_any_dtype helper to dtypes/common --- pandas/core/dtypes/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py index 14184f044ae95..3c660cfc3226b 100644 --- a/pandas/core/dtypes/common.py +++ b/pandas/core/dtypes/common.py @@ -1067,6 +1067,10 @@ def is_datetime_or_timedelta_dtype(arr_or_dtype) -> bool: return _is_dtype_type(arr_or_dtype, classes(np.datetime64, np.timedelta64)) +def is_datetime_or_timedelta_any_dtype(arr_or_dtype): + return is_datetime64_any_dtype(arr_or_dtype) or is_timedelta64_dtype(arr_or_dtype) + + # This exists to silence numpy deprecation warnings, see GH#29553 def is_numeric_v_string_like(a, b): """ From 9c06bc65500a37ae0a4b59fecf8ffa63107ffb3c Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 04:07:55 +0000 Subject: [PATCH 05/10] enact TODO: relocate PeriodArray block --- pandas/core/dtypes/cast.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 7481e0f710448..33bb440a64306 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -132,6 +132,13 @@ def maybe_downcast_to_dtype(result, dtype): elif isinstance(result, ABCDataFrame): # occurs in pivot_table doctest return result + elif is_period_dtype(dtype): + from pandas.core.arrays import PeriodArray + + with suppress(TypeError): + # e.g. TypeError: int() argument must be a string, a + # bytes-like object or a number, not 'Period + return PeriodArray(result, freq=dtype.freq) if isinstance(dtype, str): if dtype == "infer": @@ -163,7 +170,10 @@ def maybe_downcast_to_dtype(result, dtype): # a datetimelike # GH12821, iNaT is cast to float if is_datetime_or_timedelta_any_dtype(dtype) and result.dtype.kind in ["i", "f"]: - if not is_datetime_or_timedelta_dtype(dtype): + + if is_datetime_or_timedelta_dtype(dtype): + result = result.astype(dtype) + else: # not a numpy dtype if dtype.tz: # convert to datetime and change timezone @@ -171,17 +181,6 @@ def maybe_downcast_to_dtype(result, dtype): result = to_datetime(result).tz_localize("utc") result = result.tz_convert(dtype.tz) - else: - result = result.astype(dtype) - - elif is_period_dtype(dtype): - # TODO(DatetimeArray): merge with previous elif - from pandas.core.arrays import PeriodArray - - with suppress(TypeError): - # e.g. TypeError: int() argument must be a string, a - # bytes-like object or a number, not 'Period - return PeriodArray(result, freq=dtype.freq) return result From 0a23240bbe32cd868c3c4c825ca34395f404bae4 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 16:23:05 +0000 Subject: [PATCH 06/10] feedback: revert changes to dtype checks --- pandas/core/dtypes/cast.py | 7 +++---- pandas/core/dtypes/common.py | 4 ---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 33bb440a64306..c7d22906cd3e5 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -12,6 +12,7 @@ from pandas._libs.tslibs import ( NaT, OutOfBoundsDatetime, + Period, Timedelta, Timestamp, iNaT, @@ -40,7 +41,6 @@ is_datetime64_dtype, is_datetime64_ns_dtype, is_datetime64tz_dtype, - is_datetime_or_timedelta_any_dtype, is_datetime_or_timedelta_dtype, is_dtype_equal, is_extension_array_dtype, @@ -50,7 +50,6 @@ is_integer_dtype, is_numeric_dtype, is_object_dtype, - is_period_dtype, is_scalar, is_sparse, is_string_dtype, @@ -132,7 +131,7 @@ def maybe_downcast_to_dtype(result, dtype): elif isinstance(result, ABCDataFrame): # occurs in pivot_table doctest return result - elif is_period_dtype(dtype): + elif dtype.type is Period: from pandas.core.arrays import PeriodArray with suppress(TypeError): @@ -169,7 +168,7 @@ def maybe_downcast_to_dtype(result, dtype): # a datetimelike # GH12821, iNaT is cast to float - if is_datetime_or_timedelta_any_dtype(dtype) and result.dtype.kind in ["i", "f"]: + if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: if is_datetime_or_timedelta_dtype(dtype): result = result.astype(dtype) diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py index 3c660cfc3226b..14184f044ae95 100644 --- a/pandas/core/dtypes/common.py +++ b/pandas/core/dtypes/common.py @@ -1067,10 +1067,6 @@ def is_datetime_or_timedelta_dtype(arr_or_dtype) -> bool: return _is_dtype_type(arr_or_dtype, classes(np.datetime64, np.timedelta64)) -def is_datetime_or_timedelta_any_dtype(arr_or_dtype): - return is_datetime64_any_dtype(arr_or_dtype) or is_timedelta64_dtype(arr_or_dtype) - - # This exists to silence numpy deprecation warnings, see GH#29553 def is_numeric_v_string_like(a, b): """ From e2d0375d0d327d5def2e084449ac46747f483a32 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 16:55:08 +0000 Subject: [PATCH 07/10] reorder clause --- pandas/core/dtypes/cast.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index c7d22906cd3e5..7f57d675d7e44 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -131,13 +131,6 @@ def maybe_downcast_to_dtype(result, dtype): elif isinstance(result, ABCDataFrame): # occurs in pivot_table doctest return result - elif dtype.type is Period: - from pandas.core.arrays import PeriodArray - - with suppress(TypeError): - # e.g. TypeError: int() argument must be a string, a - # bytes-like object or a number, not 'Period - return PeriodArray(result, freq=dtype.freq) if isinstance(dtype, str): if dtype == "infer": @@ -162,6 +155,14 @@ def maybe_downcast_to_dtype(result, dtype): dtype = np.dtype(dtype) + elif dtype.type is Period: + from pandas.core.arrays import PeriodArray + + with suppress(TypeError): + # e.g. TypeError: int() argument must be a string, a + # bytes-like object or a number, not 'Period + return PeriodArray(result, freq=dtype.freq) + converted = maybe_downcast_numeric(result, dtype, do_round) if converted is not result: return converted From b5e53e96c86f4cdf53e25c48faaad1054592a073 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 17:36:19 +0000 Subject: [PATCH 08/10] feedback: collapse datetimelike if body --- pandas/core/dtypes/cast.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 7f57d675d7e44..0c3d58e5a22b0 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -170,17 +170,7 @@ def maybe_downcast_to_dtype(result, dtype): # a datetimelike # GH12821, iNaT is cast to float if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: - - if is_datetime_or_timedelta_dtype(dtype): - result = result.astype(dtype) - else: - # not a numpy dtype - if dtype.tz: - # convert to datetime and change timezone - from pandas import to_datetime - - result = to_datetime(result).tz_localize("utc") - result = result.tz_convert(dtype.tz) + result = result.astype(dtype) return result From d887566b6e3c9eb64c801457851d2a1e6febc717 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 18:48:34 +0000 Subject: [PATCH 09/10] revert datelike if-else body --- pandas/core/dtypes/cast.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 0c3d58e5a22b0..7c48237c8c816 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -170,7 +170,16 @@ def maybe_downcast_to_dtype(result, dtype): # a datetimelike # GH12821, iNaT is cast to float if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: - result = result.astype(dtype) + if is_datetime_or_timedelta_dtype(dtype): + result = result.astype(dtype) + else: + # not a numpy dtype + if dtype.tz: + # convert to datetime and change timezone + from pandas import to_datetime + + result = to_datetime(result).tz_localize("utc") + result = result.tz_convert(dtype.tz) return result From e46b10d96a28e3cd914bdb3039071082e640422c Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 12 Oct 2020 17:06:08 +0000 Subject: [PATCH 10/10] feedback: revert datetimelike block --- pandas/core/dtypes/cast.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 7c48237c8c816..336d33cd35d7a 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -170,9 +170,7 @@ def maybe_downcast_to_dtype(result, dtype): # a datetimelike # GH12821, iNaT is cast to float if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: - if is_datetime_or_timedelta_dtype(dtype): - result = result.astype(dtype) - else: + if hasattr(dtype, "tz"): # not a numpy dtype if dtype.tz: # convert to datetime and change timezone @@ -180,6 +178,8 @@ def maybe_downcast_to_dtype(result, dtype): result = to_datetime(result).tz_localize("utc") result = result.tz_convert(dtype.tz) + else: + result = result.astype(dtype) return result