PERF: use isna_array instead of isna in ArrayManager.fillna

Author: jorisvandenbossche

pandas/core/dtypes/missing.py

@@ -12,6 +12,7 @@
 import pandas._libs.missing as libmissing
 from pandas._libs.tslibs import (
     NaT,
+    Period,
     iNaT,
 )
 from pandas._typing import (
@@ -210,6 +211,7 @@ def _use_inf_as_na(key):
     """
     inf_as_na = get_option(key)
     globals()["_isna"] = partial(_isna, inf_as_na=inf_as_na)
+    globals()["_isna_array"] = partial(_isna_array, inf_as_na=inf_as_na)
     if inf_as_na:
         globals()["nan_checker"] = lambda x: ~np.isfinite(x)
         globals()["INF_AS_NA"] = True
@@ -244,7 +246,8 @@ def _isna_array(values: ArrayLike, inf_as_na: bool = False):
             result = values.isna()
     elif is_string_or_object_np_dtype(values.dtype):
         result = _isna_string_dtype(values, inf_as_na=inf_as_na)
-    elif needs_i8_conversion(dtype):
+    # elif needs_i8_conversion(dtype):
+    elif dtype.kind in "mM" or (dtype.kind == "O" and dtype.type is Period):
         # this is the NaT pattern
         result = values.view("i8") == iNaT
     else:

pandas/core/internals/array_manager.py

@@ -48,6 +48,7 @@
 )
 from pandas.core.dtypes.inference import is_inferred_bool_dtype
 from pandas.core.dtypes.missing import (
+    _isna_array,
     array_equals,
     isna,
     na_value_for_dtype,
@@ -192,6 +193,7 @@ def apply(
         f,
         align_keys: list[str] | None = None,
         ignore_failures: bool = False,
+        verify_integrity=True,
         **kwargs,
     ) -> T:
         """
@@ -262,7 +264,7 @@ def apply(
 
         # error: Argument 1 to "ArrayManager" has incompatible type "List[ndarray]";
         # expected "List[Union[ndarray, ExtensionArray]]"
-        return type(self)(result_arrays, new_axes)  # type: ignore[arg-type]
+        return type(self)(result_arrays, new_axes, verify_integrity=verify_integrity)  # type: ignore[arg-type]
 
     def apply_with_block(self: T, f, align_keys=None, swap_axis=True, **kwargs) -> T:
         # switch axis to follow BlockManager logic
@@ -440,6 +442,9 @@ def replace_list(
     def to_native_types(self, **kwargs):
         return self.apply(to_native_types, **kwargs)
 
+    def isna(self: T, func=None) -> T:
+        return self.apply(_isna_array, verify_integrity=False)
+
     @property
     def is_mixed_type(self) -> bool:
         return True

pandas/core/internals/base.py

@@ -135,7 +135,6 @@ def apply(
     ) -> T:
         raise AbstractMethodError(self)
 
-    @final
     def isna(self: T, func) -> T:
         return self.apply("apply", func=func)