From 5cb5cf9c0c4e923541d855a78a51f275d3149fec Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sat, 10 Oct 2020 19:34:22 +0000 Subject: [PATCH 01/11] CLN: rewrite try/except/pass using contextlib.suppress --- pandas/core/dtypes/cast.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index e550309461de4..090a3eb0eeeb1 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -2,6 +2,7 @@ Routines for casting. """ +from contextlib import suppress from datetime import date, datetime, timedelta from typing import ( TYPE_CHECKING, @@ -191,12 +192,10 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): # TODO(DatetimeArray): merge with previous elif from pandas.core.arrays import PeriodArray - try: - return PeriodArray(result, freq=dtype.freq) - except TypeError: + with suppress(TypeError): # e.g. TypeError: int() argument must be a string, a # bytes-like object or a number, not 'Period - pass + return PeriodArray(result, freq=dtype.freq) return result From 586e2c7c7574316f9de8e8dbe5d227803ca41ff9 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sat, 10 Oct 2020 20:30:58 +0000 Subject: [PATCH 02/11] use is_period_dtype to check type --- pandas/core/dtypes/cast.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 090a3eb0eeeb1..6ff1f29633cdd 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -23,7 +23,6 @@ from pandas._libs.tslibs import ( NaT, OutOfBoundsDatetime, - Period, Timedelta, Timestamp, conversion, @@ -63,6 +62,7 @@ is_integer_dtype, is_numeric_dtype, is_object_dtype, + is_period_dtype, is_scalar, is_sparse, is_string_dtype, @@ -188,7 +188,7 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): else: result = result.astype(dtype) - elif dtype.type is Period: + elif is_period_dtype(dtype): # TODO(DatetimeArray): merge with previous elif from pandas.core.arrays import PeriodArray From 0f08c523a30ec39cab8d934c941a6430d7acb688 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sat, 10 Oct 2020 22:07:21 +0000 Subject: [PATCH 03/11] clean up if condition --- pandas/core/dtypes/cast.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 6ff1f29633cdd..34692b2db6b46 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -53,6 +53,7 @@ is_datetime64_dtype, is_datetime64_ns_dtype, is_datetime64tz_dtype, + is_datetime_or_timedelta_any_dtype, is_datetime_or_timedelta_dtype, is_dtype_equal, is_extension_array_dtype, @@ -175,9 +176,9 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): return converted # a datetimelike - # GH12821, iNaT is casted to float - if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: - if hasattr(dtype, "tz"): + # GH12821, iNaT is cast to float + if is_datetime_or_timedelta_any_dtype(dtype) and result.dtype.kind in ["i", "f"]: + if not is_datetime_or_timedelta_dtype(dtype): # not a numpy dtype if dtype.tz: # convert to datetime and change timezone From 31a3d40389350c441a33c4fbf9729fc37f2dd308 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sat, 10 Oct 2020 22:21:07 +0000 Subject: [PATCH 04/11] add is_datetime_or_timedelta_any_dtype helper to dtypes/common --- pandas/core/dtypes/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py index 14184f044ae95..3c660cfc3226b 100644 --- a/pandas/core/dtypes/common.py +++ b/pandas/core/dtypes/common.py @@ -1067,6 +1067,10 @@ def is_datetime_or_timedelta_dtype(arr_or_dtype) -> bool: return _is_dtype_type(arr_or_dtype, classes(np.datetime64, np.timedelta64)) +def is_datetime_or_timedelta_any_dtype(arr_or_dtype): + return is_datetime64_any_dtype(arr_or_dtype) or is_timedelta64_dtype(arr_or_dtype) + + # This exists to silence numpy deprecation warnings, see GH#29553 def is_numeric_v_string_like(a, b): """ From 3197ff6cd351b6062b1d12b7300ffab98a32b67a Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 04:07:55 +0000 Subject: [PATCH 05/11] enact TODO: relocate PeriodArray block --- pandas/core/dtypes/cast.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 34692b2db6b46..26e450663808c 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -147,6 +147,13 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): elif isinstance(result, ABCDataFrame): # occurs in pivot_table doctest return result + elif is_period_dtype(dtype): + from pandas.core.arrays import PeriodArray + + with suppress(TypeError): + # e.g. TypeError: int() argument must be a string, a + # bytes-like object or a number, not 'Period + return PeriodArray(result, freq=dtype.freq) if isinstance(dtype, str): if dtype == "infer": @@ -178,7 +185,10 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): # a datetimelike # GH12821, iNaT is cast to float if is_datetime_or_timedelta_any_dtype(dtype) and result.dtype.kind in ["i", "f"]: - if not is_datetime_or_timedelta_dtype(dtype): + + if is_datetime_or_timedelta_dtype(dtype): + result = result.astype(dtype) + else: # not a numpy dtype if dtype.tz: # convert to datetime and change timezone @@ -186,17 +196,6 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): result = to_datetime(result).tz_localize("utc") result = result.tz_convert(dtype.tz) - else: - result = result.astype(dtype) - - elif is_period_dtype(dtype): - # TODO(DatetimeArray): merge with previous elif - from pandas.core.arrays import PeriodArray - - with suppress(TypeError): - # e.g. TypeError: int() argument must be a string, a - # bytes-like object or a number, not 'Period - return PeriodArray(result, freq=dtype.freq) return result From 04ece89fd4e2d24eb5749ad5536e6371b0a3d883 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 16:23:05 +0000 Subject: [PATCH 06/11] feedback: revert changes to dtype checks --- pandas/core/dtypes/cast.py | 7 +++---- pandas/core/dtypes/common.py | 4 ---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 26e450663808c..dd45f8cf29657 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -23,6 +23,7 @@ from pandas._libs.tslibs import ( NaT, OutOfBoundsDatetime, + Period, Timedelta, Timestamp, conversion, @@ -53,7 +54,6 @@ is_datetime64_dtype, is_datetime64_ns_dtype, is_datetime64tz_dtype, - is_datetime_or_timedelta_any_dtype, is_datetime_or_timedelta_dtype, is_dtype_equal, is_extension_array_dtype, @@ -63,7 +63,6 @@ is_integer_dtype, is_numeric_dtype, is_object_dtype, - is_period_dtype, is_scalar, is_sparse, is_string_dtype, @@ -147,7 +146,7 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): elif isinstance(result, ABCDataFrame): # occurs in pivot_table doctest return result - elif is_period_dtype(dtype): + elif dtype.type is Period: from pandas.core.arrays import PeriodArray with suppress(TypeError): @@ -184,7 +183,7 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): # a datetimelike # GH12821, iNaT is cast to float - if is_datetime_or_timedelta_any_dtype(dtype) and result.dtype.kind in ["i", "f"]: + if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: if is_datetime_or_timedelta_dtype(dtype): result = result.astype(dtype) diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py index 3c660cfc3226b..14184f044ae95 100644 --- a/pandas/core/dtypes/common.py +++ b/pandas/core/dtypes/common.py @@ -1067,10 +1067,6 @@ def is_datetime_or_timedelta_dtype(arr_or_dtype) -> bool: return _is_dtype_type(arr_or_dtype, classes(np.datetime64, np.timedelta64)) -def is_datetime_or_timedelta_any_dtype(arr_or_dtype): - return is_datetime64_any_dtype(arr_or_dtype) or is_timedelta64_dtype(arr_or_dtype) - - # This exists to silence numpy deprecation warnings, see GH#29553 def is_numeric_v_string_like(a, b): """ From d5f6d66f417eed8230201d2751fc766a8e83ef50 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 16:55:08 +0000 Subject: [PATCH 07/11] reorder clause --- pandas/core/dtypes/cast.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index dd45f8cf29657..48bccac308994 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -146,13 +146,6 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): elif isinstance(result, ABCDataFrame): # occurs in pivot_table doctest return result - elif dtype.type is Period: - from pandas.core.arrays import PeriodArray - - with suppress(TypeError): - # e.g. TypeError: int() argument must be a string, a - # bytes-like object or a number, not 'Period - return PeriodArray(result, freq=dtype.freq) if isinstance(dtype, str): if dtype == "infer": @@ -177,6 +170,14 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): dtype = np.dtype(dtype) + elif dtype.type is Period: + from pandas.core.arrays import PeriodArray + + with suppress(TypeError): + # e.g. TypeError: int() argument must be a string, a + # bytes-like object or a number, not 'Period + return PeriodArray(result, freq=dtype.freq) + converted = maybe_downcast_numeric(result, dtype, do_round) if converted is not result: return converted From 196751117f75d3ca3154fb30056fc9aefad886d2 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 17:36:19 +0000 Subject: [PATCH 08/11] feedback: collapse datetimelike if body --- pandas/core/dtypes/cast.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 48bccac308994..14a4e813fefc3 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -185,17 +185,7 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): # a datetimelike # GH12821, iNaT is cast to float if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: - - if is_datetime_or_timedelta_dtype(dtype): - result = result.astype(dtype) - else: - # not a numpy dtype - if dtype.tz: - # convert to datetime and change timezone - from pandas import to_datetime - - result = to_datetime(result).tz_localize("utc") - result = result.tz_convert(dtype.tz) + result = result.astype(dtype) return result From 5303504938310e3917c7aa2b1db3f7d303a8b3c0 Mon Sep 17 00:00:00 2001 From: arw2019 Date: Sun, 11 Oct 2020 18:48:34 +0000 Subject: [PATCH 09/11] revert datelike if-else body --- pandas/core/dtypes/cast.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 14a4e813fefc3..e4c2a1890b551 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -185,7 +185,16 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): # a datetimelike # GH12821, iNaT is cast to float if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: - result = result.astype(dtype) + if is_datetime_or_timedelta_dtype(dtype): + result = result.astype(dtype) + else: + # not a numpy dtype + if dtype.tz: + # convert to datetime and change timezone + from pandas import to_datetime + + result = to_datetime(result).tz_localize("utc") + result = result.tz_convert(dtype.tz) return result From 7619e8bf3802c63a98231da568a171fca1e4565b Mon Sep 17 00:00:00 2001 From: arw2019 Date: Mon, 12 Oct 2020 17:06:08 +0000 Subject: [PATCH 10/11] feedback: revert datetimelike block --- pandas/core/dtypes/cast.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index e4c2a1890b551..b6b9a2ae2aab8 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -185,9 +185,7 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): # a datetimelike # GH12821, iNaT is cast to float if dtype.kind in ["M", "m"] and result.dtype.kind in ["i", "f"]: - if is_datetime_or_timedelta_dtype(dtype): - result = result.astype(dtype) - else: + if hasattr(dtype, "tz"): # not a numpy dtype if dtype.tz: # convert to datetime and change timezone @@ -195,6 +193,8 @@ def maybe_downcast_to_dtype(result, dtype: Dtype): result = to_datetime(result).tz_localize("utc") result = result.tz_convert(dtype.tz) + else: + result = result.astype(dtype) return result From 048508b6c420b5bad64145e97a3df9662593420e Mon Sep 17 00:00:00 2001 From: arw2019 Date: Thu, 15 Oct 2020 03:04:37 +0000 Subject: [PATCH 11/11] TYP: add correct type hint for maybe_downcast_to_dtype --- pandas/core/dtypes/cast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index b6b9a2ae2aab8..bcd878bb5a084 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -134,7 +134,7 @@ def is_nested_object(obj) -> bool: return False -def maybe_downcast_to_dtype(result, dtype: Dtype): +def maybe_downcast_to_dtype(result, dtype: Union[str, np.dtype]): """ try to cast to the specified dtype (e.g. convert back to bool/int or could be an astype of float64->float32