From 82e90e38a599218ac70b79d12eb6434a292c9c7d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 1 Nov 2019 17:12:34 -0700 Subject: [PATCH 1/3] REF: Implement maybe_promote_scalar --- pandas/core/dtypes/cast.py | 6 ++++++ pandas/core/internals/blocks.py | 8 ++++---- pandas/core/internals/managers.py | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 3e92906be706c..8fec99c41e8e6 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -353,6 +353,12 @@ def maybe_promote(dtype, fill_value=np.nan): fill_value = np.nan dtype = np.dtype(np.object_) + return maybe_promote_scalar(dtype, fill_value) + + +def maybe_promote_scalar(dtype, fill_value=np.nan): + # maybe_promote, but restricted to fill_value being non-ndarray + # returns tuple of (dtype, fill_value) if issubclass(dtype.type, np.datetime64): if isinstance(fill_value, datetime) and fill_value.tzinfo is not None: diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 51108d9a5a573..b494aab9dda27 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -22,7 +22,7 @@ maybe_downcast_numeric, maybe_downcast_to_dtype, maybe_infer_dtype_type, - maybe_promote, + maybe_promote_scalar, maybe_upcast, soft_convert_objects, ) @@ -856,7 +856,7 @@ def setitem(self, indexer, value): # cast the values to a type that can hold nan (if necessary) if not self._can_hold_element(value): - dtype, _ = maybe_promote(arr_value.dtype) + dtype, _ = maybe_promote_scalar(arr_value.dtype) values = values.astype(dtype) if transpose: @@ -994,7 +994,7 @@ def f(mask, val, idx): n = np.array(new) # type of the new block - dtype, _ = maybe_promote(n.dtype) + dtype, _ = maybe_promote_scalar(n.dtype) # we need to explicitly astype here to make a copy n = n.astype(dtype) @@ -3169,7 +3169,7 @@ def _putmask_preserve(nv, n): return _putmask_preserve(v, n) # change the dtype if needed - dtype, _ = maybe_promote(n.dtype) + dtype, _ = maybe_promote_scalar(n.dtype) if is_extension_type(v.dtype) and is_object_dtype(dtype): v = v._internal_get_values(dtype) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index c47aaf7c773c4..d6b41a884a15d 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -14,7 +14,7 @@ find_common_type, infer_dtype_from_scalar, maybe_convert_objects, - maybe_promote, + maybe_promote_scalar, ) from pandas.core.dtypes.common import ( _NS_DTYPE, @@ -1292,7 +1292,7 @@ def _slice_take_blocks_ax0(self, slice_or_indexer, fill_tuple=None): return [blk.getitem_block(slobj, new_mgr_locs=slice(0, sllen))] elif not allow_fill or self.ndim == 1: if allow_fill and fill_tuple[0] is None: - _, fill_value = maybe_promote(blk.dtype) + _, fill_value = maybe_promote_scalar(blk.dtype) fill_tuple = (fill_value,) return [ From 9fae53bcc1a6cdd712f32ff81d1ad3574d12977e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 2 Nov 2019 08:59:44 -0700 Subject: [PATCH 2/3] docstring --- pandas/core/dtypes/cast.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 8fec99c41e8e6..8d5ccaf86185c 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -357,7 +357,22 @@ def maybe_promote(dtype, fill_value=np.nan): def maybe_promote_scalar(dtype, fill_value=np.nan): - # maybe_promote, but restricted to fill_value being non-ndarray + """ + Find the minimal dtype that can hold both the given dtype and fill_value. + + Parameters + ---------- + dtype : np.dtype or ExceptionDtype + fill_value : scalar, default np.nan + + Returns + ------- + dtype + Upcasted from dtype argument if necessary. + fill_value + Upcasted from fill_value argument if necessary. + """ + assert is_scalar(fill_value) # returns tuple of (dtype, fill_value) if issubclass(dtype.type, np.datetime64): From 3e5b72208c0251fc9c76ddc952f522a7fb74cb6c Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 2 Nov 2019 11:59:31 -0700 Subject: [PATCH 3/3] revert assertion --- pandas/core/dtypes/cast.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 9bde5298a67e6..b34f25267184a 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -376,7 +376,6 @@ def maybe_promote_scalar(dtype, fill_value=np.nan): fill_value Upcasted from fill_value argument if necessary. """ - assert is_scalar(fill_value) # returns tuple of (dtype, fill_value) if issubclass(dtype.type, np.datetime64):