Disallow non-scalar fill_value in maybe_upcast (pandas-dev#29362)

Author: jbrockmendel

pandas/core/dtypes/cast.py

@@ -686,6 +686,8 @@ def maybe_upcast(values, fill_value=np.nan, dtype=None, copy=False):
     dtype : if None, then use the dtype of the values, else coerce to this type
     copy : if True always make a copy even if no upcast is required
     """
+    if not is_scalar(fill_value):
+        raise ValueError("fill_value must be a scalar")
 
     if is_extension_type(values):
         if copy:

pandas/core/internals/blocks.py

@@ -1283,6 +1283,10 @@ def diff(self, n: int, axis: int = 1) -> List["Block"]:
     def shift(self, periods, axis=0, fill_value=None):
         """ shift the block by periods, possibly upcast """
 
+        if not lib.is_scalar(fill_value):
+            # We could go further and require e.g. self._can_hold_element(fv)
+            raise ValueError("fill_value must be a scalar")
+
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also
         new_values, fill_value = maybe_upcast(self.values, fill_value)

pandas/core/internals/construction.py

@@ -97,6 +97,9 @@ def masked_rec_array_to_mgr(data, index, columns, dtype, copy):
     # fill if needed
     new_arrays = []
     for fv, arr, col in zip(fill_value, arrays, arr_columns):
+        # TODO: numpy docs suggest fv must be scalar, but could it be
+        #  non-scalar for object dtype?
+        assert lib.is_scalar(fv), fv
         mask = ma.getmaskarray(data[col])
         if mask.any():
             arr, fv = maybe_upcast(arr, fill_value=fv, copy=True)