diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index 1ae76984484af..6b398c88f96f3 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -212,6 +212,7 @@ Indexing - Bug in :meth:`DataFrame.iloc` where indexing a single row on a :class:`DataFrame` with a single ExtensionDtype column gave a copy instead of a view on the underlying data (:issue:`45241`) - Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised insead of casting to a common dtype (:issue:`45070`) - Bug when setting a value too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`, :issue:`32878`) +- Bug in :meth:`Series.__setitem__` where setting :attr:`NA` into a numeric-dtpye :class:`Series` would incorrectly upcast to object-dtype rather than treating the value as ``np.nan`` (:issue:`44199`) - Missing diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 88b848f34fbe5..a3cdd7d9f1041 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -8882,9 +8882,9 @@ def applymap( >>> df_copy = df.copy() >>> df_copy.iloc[0, 0] = pd.NA >>> df_copy.applymap(lambda x: len(str(x)), na_action='ignore') - 0 1 - 0 4 - 1 5 5 + 0 1 + 0 NaN 4 + 1 5.0 5 Note that a vectorized version of `func` often exists, which will be much faster. You could square each number elementwise. diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 717abb998dd65..aaf1d8c359868 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -5210,7 +5210,8 @@ def putmask(self, mask, value) -> Index: if noop: return self.copy() - if value is None and (self._is_numeric_dtype or self.dtype == object): + if self.dtype != object and is_valid_na_for_dtype(value, self.dtype): + # e.g. None -> np.nan, see also Block._standardize_fill_value value = self._na_value try: converted = self._validate_fill_value(value) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index ef1cd92a60540..32e3ec676ce80 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -879,6 +879,12 @@ def _replace_coerce( # --------------------------------------------------------------------- + def _standardize_fill_value(self, value): + # if we are passed a scalar None, convert it here + if self.dtype != _dtype_obj and is_valid_na_for_dtype(value, self.dtype): + value = self.fill_value + return value + def _maybe_squeeze_arg(self, arg: np.ndarray) -> np.ndarray: """ For compatibility with 1D-only ExtensionArrays. @@ -916,10 +922,7 @@ def setitem(self, indexer, value): if isinstance(indexer, np.ndarray) and indexer.ndim > self.ndim: raise ValueError(f"Cannot set values with ndim > {self.ndim}") - # coerce None values, if appropriate - if value is None: - if self.is_numeric: - value = np.nan + value = self._standardize_fill_value(value) # coerce if block dtype can store value values = cast(np.ndarray, self.values) @@ -980,9 +983,7 @@ def putmask(self, mask, new) -> list[Block]: if new is lib.no_default: new = self.fill_value - # if we are passed a scalar None, convert it here - if not self.is_object and is_valid_na_for_dtype(new, self.dtype): - new = self.fill_value + new = self._standardize_fill_value(new) if self._can_hold_element(new): putmask_without_repeat(values.T, mask, new) @@ -1164,8 +1165,7 @@ def shift(self, periods: int, axis: int = 0, fill_value: Any = None) -> list[Blo # see test_shift_object_non_scalar_fill raise ValueError("fill_value must be a scalar") - if is_valid_na_for_dtype(fill_value, self.dtype) and self.dtype != _dtype_obj: - fill_value = self.fill_value + fill_value = self._standardize_fill_value(fill_value) if not self._can_hold_element(fill_value): nb = self.coerce_to_target_dtype(fill_value) @@ -1208,8 +1208,7 @@ def where(self, other, cond) -> list[Block]: if other is lib.no_default: other = self.fill_value - if is_valid_na_for_dtype(other, self.dtype) and self.dtype != _dtype_obj: - other = self.fill_value + other = self._standardize_fill_value(other) if not self._can_hold_element(other): # we cannot coerce, return a compat dtype diff --git a/pandas/tests/series/indexing/test_setitem.py b/pandas/tests/series/indexing/test_setitem.py index 38952ddfca2bb..e7261d75dfd30 100644 --- a/pandas/tests/series/indexing/test_setitem.py +++ b/pandas/tests/series/indexing/test_setitem.py @@ -9,6 +9,7 @@ from pandas.core.dtypes.common import is_list_like from pandas import ( + NA, Categorical, DataFrame, DatetimeIndex, @@ -770,11 +771,13 @@ def test_index_putmask(self, obj, key, expected, val): ], ) class TestSetitemCastingEquivalents(SetitemCastingEquivalents): - @pytest.fixture(params=[np.nan, np.float64("NaN")]) + @pytest.fixture(params=[np.nan, np.float64("NaN"), None, NA]) def val(self, request): """ - One python float NaN, one np.float64. Only np.float64 has a `dtype` - attribute. + NA values that should generally be valid_na for *all* dtypes. + + Include both python float NaN and np.float64; only np.float64 has a + `dtype` attribute. """ return request.param