Revert "CLN: remove unused concat code (pandas-dev#43577)"

This reverts commit 95eb153.

Author: jorisvandenbossche

pandas/core/dtypes/missing.py

@@ -18,6 +18,7 @@
 import pandas._libs.missing as libmissing
 from pandas._libs.tslibs import (
     NaT,
+    Period,
     iNaT,
 )
 
@@ -739,3 +740,40 @@ def is_valid_na_for_dtype(obj, dtype: DtypeObj) -> bool:
 
     # fallback, default to allowing NaN, None, NA, NaT
     return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal))
+
+
+def isna_all(arr: ArrayLike) -> bool:
+    """
+    Optimized equivalent to isna(arr).all()
+    """
+    total_len = len(arr)
+
+    # Usually it's enough to check but a small fraction of values to see if
+    #  a block is NOT null, chunks should help in such cases.
+    #  parameters 1000 and 40 were chosen arbitrarily
+    chunk_len = max(total_len // 40, 1000)
+
+    dtype = arr.dtype
+    if dtype.kind == "f":
+        checker = nan_checker
+
+    elif dtype.kind in ["m", "M"] or dtype.type is Period:
+        # error: Incompatible types in assignment (expression has type
+        # "Callable[[Any], Any]", variable has type "ufunc")
+        checker = lambda x: np.asarray(x.view("i8")) == iNaT  # type: ignore[assignment]
+
+    else:
+        # error: Incompatible types in assignment (expression has type "Callable[[Any],
+        # Any]", variable has type "ufunc")
+        checker = lambda x: _isna_array(  # type: ignore[assignment]
+            x, inf_as_na=INF_AS_NA
+        )
+
+    return all(
+        # error: Argument 1 to "__call__" of "ufunc" has incompatible type
+        # "Union[ExtensionArray, Any]"; expected "Union[Union[int, float, complex, str,
+        # bytes, generic], Sequence[Union[int, float, complex, str, bytes, generic]],
+        # Sequence[Sequence[Any]], _SupportsArray]"
+        checker(arr[i : i + chunk_len]).all()  # type: ignore[arg-type]
+        for i in range(0, total_len, chunk_len)
+    )

pandas/core/internals/concat.py

@@ -10,7 +10,11 @@
 
 import numpy as np
 
-from pandas._libs import internals as libinternals
+from pandas._libs import (
+    NaT,
+    internals as libinternals,
+)
+from pandas._libs.missing import NA
 from pandas._typing import (
     ArrayLike,
     DtypeObj,
@@ -27,12 +31,14 @@
     is_1d_only_ea_dtype,
     is_datetime64tz_dtype,
     is_dtype_equal,
+    needs_i8_conversion,
 )
 from pandas.core.dtypes.concat import (
     cast_to_common_type,
     concat_compat,
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype
+from pandas.core.dtypes.missing import is_valid_na_for_dtype
 
 import pandas.core.algorithms as algos
 from pandas.core.arrays import (
@@ -378,6 +384,36 @@ def dtype(self):
             return blk.dtype
         return ensure_dtype_can_hold_na(blk.dtype)
 
+    def _is_valid_na_for(self, dtype: DtypeObj) -> bool:
+        """
+        Check that we are all-NA of a type/dtype that is compatible with this dtype.
+        Augments `self.is_na` with an additional check of the type of NA values.
+        """
+        if not self.is_na:
+            return False
+        if self.block.dtype.kind == "V":
+            return True
+
+        if self.dtype == object:
+            values = self.block.values
+            return all(is_valid_na_for_dtype(x, dtype) for x in values.ravel(order="K"))
+
+        na_value = self.block.fill_value
+        if na_value is NaT and not is_dtype_equal(self.dtype, dtype):
+            # e.g. we are dt64 and other is td64
+            # fill_values match but we should not cast self.block.values to dtype
+            # TODO: this will need updating if we ever have non-nano dt64/td64
+            return False
+
+        if na_value is NA and needs_i8_conversion(dtype):
+            # FIXME: kludge; test_append_empty_frame_with_timedelta64ns_nat
+            #  e.g. self.dtype == "Int64" and dtype is td64, we dont want
+            #  to consider these as matching
+            return False
+
+        # TODO: better to use can_hold_element?
+        return is_valid_na_for_dtype(na_value, dtype)
+
     @cache_readonly
     def is_na(self) -> bool:
         blk = self.block
@@ -388,14 +424,24 @@ def is_na(self) -> bool:
     def get_reindexed_values(self, empty_dtype: DtypeObj, upcasted_na) -> ArrayLike:
         values: ArrayLike
 
-        if upcasted_na is None and not self.is_na:
+        if upcasted_na is None and self.block.dtype.kind != "V":
             # No upcasting is necessary
             fill_value = self.block.fill_value
             values = self.block.get_values()
         else:
             fill_value = upcasted_na
 
-            if self.is_na:
+            if self._is_valid_na_for(empty_dtype):
+                # note: always holds when self.block.dtype.kind == "V"
+                blk_dtype = self.block.dtype
+
+                if blk_dtype == np.dtype("object"):
+                    # we want to avoid filling with np.nan if we are
+                    # using None; we already know that we are all
+                    # nulls
+                    values = self.block.values.ravel(order="K")
+                    if len(values) and values[0] is None:
+                        fill_value = None
 
                 if is_datetime64tz_dtype(empty_dtype):
                     i8values = np.full(self.shape, fill_value.value)
@@ -464,7 +510,8 @@ def _concatenate_join_units(
 
     empty_dtype = _get_empty_dtype(join_units)
 
-    upcasted_na = _dtype_to_na_value(empty_dtype)
+    has_none_blocks = any(unit.block.dtype.kind == "V" for unit in join_units)
+    upcasted_na = _dtype_to_na_value(empty_dtype, has_none_blocks)
 
     to_concat = [
         ju.get_reindexed_values(empty_dtype=empty_dtype, upcasted_na=upcasted_na)
@@ -506,7 +553,7 @@ def _concatenate_join_units(
     return concat_values
 
 
-def _dtype_to_na_value(dtype: DtypeObj):
+def _dtype_to_na_value(dtype: DtypeObj, has_none_blocks: bool):
     """
     Find the NA value to go with this dtype.
     """
@@ -544,9 +591,11 @@ def _get_empty_dtype(join_units: Sequence[JoinUnit]) -> DtypeObj:
         empty_dtype = join_units[0].block.dtype
         return empty_dtype
 
-    has_none_blocks = any(unit.is_na for unit in join_units)
+    has_none_blocks = any(unit.block.dtype.kind == "V" for unit in join_units)
 
     dtypes = [unit.dtype for unit in join_units if not unit.is_na]
+    if not len(dtypes):
+        dtypes = [unit.dtype for unit in join_units if unit.block.dtype.kind != "V"]
 
     dtype = find_common_type(dtypes)
     if has_none_blocks: