Merge changes from PR#53089

Rewrite to use ideas from 53089 to handle scalar objects that are different in size than the objects in the EA.

There are some parse/eval failures that I'm pretty sure have nothing to do with these changes, but due instead to a moment in time when I merged with the HEAD of the development branch.

Signed-off-by: Michael Tiemann <[email protected]>

Author: MichaelTiemannOSC

pandas/core/arrays/_mixins.py

@@ -517,9 +517,7 @@ def _cast_quantile_result(self, res_values: np.ndarray) -> np.ndarray:
     # numpy-like methods
 
     @classmethod
-    def _empty(
-        cls, shape: Shape, dtype: ExtensionDtype, fill_value: object = None
-    ) -> Self:
+    def _empty(cls, shape: Shape, dtype: ExtensionDtype) -> Self:
         """
         Analogous to np.empty(shape, dtype=dtype)
 

pandas/core/arrays/arrow/array.py

@@ -251,14 +251,7 @@ def __init__(self, values: pa.Array | pa.ChunkedArray) -> None:
         self._dtype = ArrowDtype(self._pa_array.type)
 
     @classmethod
-    def _from_sequence(
-        cls,
-        scalars,
-        *,
-        dtype: Dtype | None = None,
-        fill_value: object | None = None,
-        copy: bool = False,
-    ):
+    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
         """
         Construct a new ExtensionArray from a sequence of scalars.
         """

pandas/core/arrays/base.py

@@ -85,6 +85,7 @@
         AstypeArg,
         AxisInt,
         Dtype,
+        DtypeObj,
         FillnaOptions,
         InterpolateOptions,
         NumpySorter,
@@ -262,14 +263,7 @@ class ExtensionArray:
     # ------------------------------------------------------------------------
 
     @classmethod
-    def _from_sequence(
-        cls,
-        scalars,
-        *,
-        dtype: Dtype | None = None,
-        fill_value: object = None,
-        copy: bool = False,
-    ):
+    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
         """
         Construct a new ExtensionArray from a sequence of scalars.
 
@@ -297,6 +291,35 @@ def _from_sequence(
         """
         raise AbstractMethodError(cls)
 
+    @classmethod
+    def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
+        """
+        Strict analogue to _from_sequence, allowing only sequences of scalars
+        that should be specifically inferred to the given dtype.
+        Parameters
+        ----------
+        scalars : sequence
+        dtype : ExtensionDtype
+        Raises
+        ------
+        TypeError or ValueError
+        Notes
+        -----
+        This is called in a try/except block when casting the result of a
+        pointwise operation.
+        """
+        try:
+            return cls._from_sequence(scalars, dtype=dtype, copy=False)
+        except (ValueError, TypeError):
+            raise
+        except Exception:
+            warnings.warn(
+                "_from_scalars should only raise ValueError or TypeError. "
+                "Consider overriding _from_scalars where appropriate.",
+                stacklevel=find_stack_level(),
+            )
+            raise
+
     @classmethod
     def _from_sequence_of_strings(
         cls, strings, *, dtype: Dtype | None = None, copy: bool = False
@@ -2092,7 +2115,7 @@ def _rank(
         )
 
     @classmethod
-    def _empty(cls, shape: Shape, dtype: ExtensionDtype, fill_value: object = None):
+    def _empty(cls, shape: Shape, dtype: ExtensionDtype):
         """
         Create an ExtensionArray with the given shape and dtype.
 
@@ -2104,7 +2127,7 @@ def _empty(cls, shape: Shape, dtype: ExtensionDtype, fill_value: object = None):
         # Implementer note: while ExtensionDtype.empty is the public way to
         # call this method, it is still required to implement this `_empty`
         # method as well (it is called internally in pandas)
-        obj = cls._from_sequence([], dtype=dtype, fill_value=fill_value)
+        obj = cls._from_sequence([], dtype=dtype)
 
         taker = np.broadcast_to(np.intp(-1), shape)
         result = obj.take(taker, allow_fill=True)

pandas/core/arrays/boolean.py

@@ -5,13 +5,15 @@
     TYPE_CHECKING,
     cast,
 )
+import warnings
 
 import numpy as np
 
 from pandas._libs import (
     lib,
     missing as libmissing,
 )
+from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import is_list_like
 from pandas.core.dtypes.dtypes import register_extension_dtype
@@ -318,6 +320,36 @@ def __init__(
     def dtype(self) -> BooleanDtype:
         return self._dtype
 
+    @classmethod
+    def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
+        """
+        Strict analogue to _from_sequence, allowing only sequences of scalars
+        that should be specifically inferred to the given dtype.
+        Parameters
+        ----------
+        scalars : sequence
+        dtype : ExtensionDtype
+        Raises
+        ------
+        TypeError or ValueError
+        Notes
+        -----
+        This is called in a try/except block when casting the result of a
+        pointwise operation.
+        """
+        try:
+            # DtypeObj is bool, which conflicts with test assertion dtype=='boolean'
+            return cls._from_sequence(scalars, dtype=BooleanDtype(), copy=False)
+        except (ValueError, TypeError):
+            raise
+        except Exception:
+            warnings.warn(
+                "_from_scalars should only raise ValueError or TypeError. "
+                "Consider overriding _from_scalars where appropriate.",
+                stacklevel=find_stack_level(),
+            )
+            raise
+
     @classmethod
     def _from_sequence_of_strings(
         cls,

pandas/core/arrays/categorical.py

@@ -101,6 +101,7 @@
         AstypeArg,
         AxisInt,
         Dtype,
+        DtypeObj,
         NpDtype,
         Ordered,
         Self,
@@ -505,15 +506,26 @@ def _internal_fill_value(self) -> int:
 
     @classmethod
     def _from_sequence(
-        cls,
-        scalars,
-        *,
-        dtype: Dtype | None = None,
-        fill_value: object = None,
-        copy: bool = False,
+        cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
     ) -> Self:
         return cls(scalars, dtype=dtype, copy=copy)
 
+    @classmethod
+    def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
+        if dtype is None:
+            # The _from_scalars strictness doesn't make much sense in this case.
+            raise NotImplementedError
+
+        res = cls._from_sequence(scalars, dtype=dtype)
+
+        # if there are any non-category elements in scalars, these will be
+        #  converted to NAs in res.
+        mask = isna(scalars)
+        if not (mask == res.isna()).all():
+            # Some non-category element in scalars got converted to NA in res.
+            raise ValueError
+        return res
+
     @overload
     def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray:
         ...
@@ -1805,7 +1817,7 @@ def value_counts(self, dropna: bool = True) -> Series:
     # "ExtensionDtype"
     @classmethod
     def _empty(  # type: ignore[override]
-        cls, shape: Shape, dtype: CategoricalDtype, fill_value: object = None
+        cls, shape: Shape, dtype: CategoricalDtype
     ) -> Self:
         """
         Analogous to np.empty(shape, dtype=dtype)

pandas/core/arrays/datetimes.py

@@ -74,6 +74,7 @@
 
     from pandas._typing import (
         DateTimeErrorChoices,
+        DtypeObj,
         IntervalClosedType,
         Self,
         TimeAmbiguous,
@@ -293,11 +294,17 @@ def _simple_new(  # type: ignore[override]
         return result
 
     @classmethod
-    def _from_sequence(
-        cls, scalars, *, dtype=None, fill_value: object = None, copy: bool = False
-    ):
+    def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False):
         return cls._from_sequence_not_strict(scalars, dtype=dtype, copy=copy)
 
+    @classmethod
+    def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
+        if lib.infer_dtype(scalars, skipna=True) not in ["datetime", "datetime64"]:
+            # TODO: require any NAs be valid-for-DTA
+            # TODO: if dtype is passed, check for tzawareness compat?
+            raise ValueError
+        return cls._from_sequence(scalars, dtype=dtype)
+
     @classmethod
     def _from_sequence_not_strict(
         cls,

pandas/core/arrays/interval.py

@@ -382,7 +382,6 @@ def _from_sequence(
         scalars,
         *,
         dtype: Dtype | None = None,
-        fill_value: object = None,
         copy: bool = False,
     ) -> Self:
         return cls(scalars, dtype=dtype, copy=copy)

pandas/core/arrays/masked.py

@@ -145,15 +145,13 @@ def __init__(
         self._mask = mask
 
     @classmethod
-    def _from_sequence(
-        cls, scalars, *, dtype=None, fill_value: object = None, copy: bool = False
-    ) -> Self:
+    def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False) -> Self:
         values, mask = cls._coerce_to_array(scalars, dtype=dtype, copy=copy)
         return cls(values, mask)
 
     @classmethod
     @doc(ExtensionArray._empty)
-    def _empty(cls, shape: Shape, dtype: ExtensionDtype, fill_value: object = None):
+    def _empty(cls, shape: Shape, dtype: ExtensionDtype):
         values = np.empty(shape, dtype=dtype.type)
         values.fill(cls._internal_fill_value)
         mask = np.ones(shape, dtype=bool)

pandas/core/arrays/numpy_.py

@@ -117,12 +117,7 @@ def __init__(
 
     @classmethod
     def _from_sequence(
-        cls,
-        scalars,
-        *,
-        dtype: Dtype | None = None,
-        fill_value: object = None,
-        copy: bool = False,
+        cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
     ) -> NumpyExtensionArray:
         if isinstance(dtype, NumpyEADtype):
             dtype = dtype._dtype

pandas/core/arrays/period.py

@@ -276,7 +276,6 @@ def _from_sequence(
         scalars,
         *,
         dtype: Dtype | None = None,
-        fill_value: object | None = None,
         copy: bool = False,
     ) -> Self:
         if dtype is not None:

pandas/core/arrays/sparse/array.py

@@ -583,14 +583,7 @@ def __setitem__(self, key, value) -> None:
         raise TypeError(msg)
 
     @classmethod
-    def _from_sequence(
-        cls,
-        scalars,
-        *,
-        dtype: Dtype | None = None,
-        fill_value: object | None = None,
-        copy: bool = False,
-    ):
+    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
         return cls(scalars, dtype=dtype)
 
     @classmethod

pandas/core/arrays/string_.py

@@ -54,9 +54,11 @@
     from pandas._typing import (
         AxisInt,
         Dtype,
+        DtypeObj,
         NumpySorter,
         NumpyValueArrayLike,
         Scalar,
+        Self,
         npt,
         type_t,
     )
@@ -228,6 +230,13 @@ def tolist(self):
             return [x.tolist() for x in self]
         return list(self.to_numpy())
 
+    @classmethod
+    def _from_scalars(cls, scalars, dtype: DtypeObj) -> Self:
+        if lib.infer_dtype(scalars, skipna=True) != "string":
+            # TODO: require any NAs be valid-for-string
+            raise ValueError
+        return cls._from_sequence(scalars, dtype=dtype)
+
 
 # error: Definition of "_concat_same_type" in base class "NDArrayBacked" is
 # incompatible with definition in base class "ExtensionArray"
@@ -340,14 +349,7 @@ def _validate(self):
             lib.convert_nans_to_NA(self._ndarray)
 
     @classmethod
-    def _from_sequence(
-        cls,
-        scalars,
-        *,
-        dtype: Dtype | None = None,
-        fill_value: object | None = None,
-        copy: bool = False,
-    ):
+    def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
         if dtype and not (isinstance(dtype, str) and dtype == "string"):
             dtype = pandas_dtype(dtype)
             assert isinstance(dtype, StringDtype) and dtype.storage == "python"
@@ -384,7 +386,7 @@ def _from_sequence_of_strings(
         return cls._from_sequence(strings, dtype=dtype, copy=copy)
 
     @classmethod
-    def _empty(cls, shape, dtype, fill_value=None) -> StringArray:
+    def _empty(cls, shape, dtype) -> StringArray:
         values = np.empty(shape, dtype=object)
         values[:] = libmissing.NA
         return cls(values).astype(dtype, copy=False)

pandas/core/arrays/string_arrow.py

@@ -137,13 +137,7 @@ def __len__(self) -> int:
         return len(self._pa_array)
 
     @classmethod
-    def _from_sequence(
-        cls,
-        scalars,
-        dtype: Dtype | None = None,
-        fill_value: object | None = None,
-        copy: bool = False,
-    ):
+    def _from_sequence(cls, scalars, dtype: Dtype | None = None, copy: bool = False):
         from pandas.core.arrays.masked import BaseMaskedArray
 
         _chk_pyarrow_available()

pandas/core/arrays/timedeltas.py

@@ -229,9 +229,7 @@ def _simple_new(  # type: ignore[override]
         return result
 
     @classmethod
-    def _from_sequence(
-        cls, data, *, dtype=None, fill_value: object = None, copy: bool = False
-    ) -> Self:
+    def _from_sequence(cls, data, *, dtype=None, copy: bool = False) -> Self:
         if dtype:
             dtype = _validate_td64_dtype(dtype)