maybe_promote: Restrict fill_value to scalar for non-object dtype (pandas-dev#29416)

Author: jbrockmendel

pandas/core/dtypes/cast.py

@@ -339,6 +339,11 @@ def changeit():
 
 
 def maybe_promote(dtype, fill_value=np.nan):
+    if not is_scalar(fill_value) and not is_object_dtype(dtype):
+        # with object dtype there is nothing to promote, and the user can
+        #  pass pretty much any weird fill_value they like
+        raise ValueError("fill_value must be a scalar")
+
     # if we passed an array here, determine the fill value by dtype
     if isinstance(fill_value, np.ndarray):
         if issubclass(fill_value.dtype.type, (np.datetime64, np.timedelta64)):
@@ -686,7 +691,8 @@ def maybe_upcast(values, fill_value=np.nan, dtype=None, copy=False):
     dtype : if None, then use the dtype of the values, else coerce to this type
     copy : if True always make a copy even if no upcast is required
     """
-    if not is_scalar(fill_value):
+    if not is_scalar(fill_value) and not is_object_dtype(values.dtype):
+        # We allow arbitrary fill values for object dtype
         raise ValueError("fill_value must be a scalar")
 
     if is_extension_type(values):

pandas/core/internals/blocks.py

@@ -1283,10 +1283,6 @@ def diff(self, n: int, axis: int = 1) -> List["Block"]:
     def shift(self, periods, axis=0, fill_value=None):
         """ shift the block by periods, possibly upcast """
 
-        if not lib.is_scalar(fill_value):
-            # We could go further and require e.g. self._can_hold_element(fv)
-            raise ValueError("fill_value must be a scalar")
-
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also
         new_values, fill_value = maybe_upcast(self.values, fill_value)