BUG: Fix merging non-indexes causes Index dtype promotion in when keys are

missing from left or right side. (GH28220)
Also closes GH24897, GH24212, and GH17257

Author: dworvos

doc/source/whatsnew/v0.25.2.rst

@@ -85,7 +85,7 @@ Groupby/resample/rolling
 Reshaping
 ^^^^^^^^^
 
--
+- Bug in certain joins when missing keys when not using left_index when how='right', or right_index when how='left' causing NA values in index and dtype promotion (:issue:`28220`).
 -
 -
 -

pandas/core/reshape/merge.py

@@ -30,10 +30,12 @@
     is_int64_dtype,
     is_integer,
     is_integer_dtype,
+    is_interval_dtype,
     is_list_like,
     is_number,
     is_numeric_dtype,
     is_object_dtype,
+    is_period_dtype,
     needs_i8_conversion,
 )
 from pandas.core.dtypes.missing import isnull, na_value_for_dtype
@@ -842,65 +844,187 @@ def _get_join_info(self):
         else:
             (left_indexer, right_indexer) = self._get_join_indexers()
 
+            if self.left_index:
+                left = self.left.index
+            else:
+                left = self.left_join_keys
+
             if self.right_index:
-                if len(self.left) > 0:
-                    join_index = self._create_join_index(
-                        self.left.index,
-                        self.right.index,
-                        left_indexer,
-                        right_indexer,
-                        how="right",
-                    )
-                else:
-                    join_index = self.right.index.take(right_indexer)
-                    left_indexer = np.array([-1] * len(join_index))
-            elif self.left_index:
-                if len(self.right) > 0:
-                    join_index = self._create_join_index(
-                        self.right.index,
-                        self.left.index,
-                        right_indexer,
-                        left_indexer,
-                        how="left",
-                    )
-                else:
-                    join_index = self.left.index.take(left_indexer)
-                    right_indexer = np.array([-1] * len(join_index))
+                right = self.right.index
+            else:
+                right = self.right_join_keys
+
+            if self.how != "right":
+                join_index = self._create_join_index(
+                    left,
+                    right,
+                    left_indexer,
+                    right_indexer,
+                    self.join_names,
+                    self.join_names,
+                )
             else:
-                join_index = Index(np.arange(len(left_indexer)))
+                join_index = self._create_join_index(
+                    right,
+                    left,
+                    right_indexer,
+                    left_indexer,
+                    self.join_names,
+                    self.join_names,
+                )
 
         if len(join_index) == 0:
             join_index = join_index.astype(object)
         return join_index, left_indexer, right_indexer
 
     def _create_join_index(
-        self, index, other_index, indexer, other_indexer, how="left"
+        self, index, other_index, indexer, other_indexer, names, other_names
     ):
         """
         Create a join index by rearranging one index to match another
 
         Parameters
         ----------
-        index: Index being rearranged
-        other_index: Index used to supply values not found in index
-        indexer: how to rearrange index
-        how: replacement is only necessary if indexer based on other_index
+        index: [Multi]Index being rearranged
+        other_index: [Multi]Index used to supply values not found in index
+        indexer: indicies for fancy indexing in index
+        other_indexer: indicies for fancy indexing in other_index
+        names: index name(s)
+        other_names: other_index name(s)
 
         Returns
         -------
         join_index
         """
-        if self.how in (how, "outer") and not isinstance(other_index, MultiIndex):
-            # if final index requires values in other_index but not target
-            # index, indexer may hold missing (-1) values, causing Index.take
-            # to take the final value in target index. So, we set the last
-            # element to be the desired fill value. We do not use allow_fill
-            # and fill_value because it throws a ValueError on integer indices
-            mask = indexer == -1
-            if np.any(mask):
-                fill_value = na_value_for_dtype(index.dtype, compat=False)
-                index = index.append(Index([fill_value]))
-        return index.take(indexer)
+
+        if not issubclass(type(index), Index) and is_list_like(index):
+            if len(index) > 1:
+                index = MultiIndex.from_arrays(index, names=names)
+            else:
+                index = Index(index[0], name=names[0])
+
+        infer_index_type = False
+        if not issubclass(type(other_index), Index) and is_list_like(other_index):
+            # must infer dtype from index if possible
+            infer_index_type = True
+
+            if len(other_index) > 1:
+                other_index = MultiIndex.from_arrays(other_index, names=other_names)
+            else:
+                other_index = Index(other_index[0], name=other_names[0])
+
+        # if final index requires values in other_index but not target
+        # index, indexer may hold missing (-1) values, causing Index.take
+        # to take the final value in target index
+        mask = indexer == -1
+
+        idx = None
+        convert_type = None
+        if self.how == "outer":
+
+            # we may need to reshape the index to include missing keys
+            outer_mask = mask & (other_indexer != -1)
+
+            if outer_mask.any():
+
+                if is_categorical_dtype(other_index.dtype):
+                    np_type = other_index.categories.dtype
+                    convert_type = other_index.categories.dtype
+                elif (
+                    is_period_dtype(other_index.dtype)
+                    or is_datetime64tz_dtype(other_index.dtype)
+                    or is_interval_dtype(other_index.dtype)
+                ):
+                    # TODO: better way to get numpy dtype without try/except?
+                    np_type = other_index.to_numpy().dtype
+                    convert_type = other_index.dtype
+                else:
+                    np_type = other_index.dtype
+
+                outer_index = np.empty(len(outer_mask), dtype=np_type)
+                outer_index[mask] = other_index.take(other_indexer[outer_mask])
+
+                if not index.empty:
+                    if is_datetime64tz_dtype(other_index.dtype):
+                        idx = np.where(
+                            outer_mask, outer_index, index.take(indexer).to_numpy()
+                        )
+                    else:
+                        idx = np.where(outer_mask, outer_index, index.take(indexer))
+                else:
+                    idx = outer_index
+
+        elif self.how == "inner":
+            if not self.left_index and not self.right_index:
+                # TODO: this behaviour below is assumed in a lot of test cases
+                return np.arange(np.sum(~mask))
+            else:
+                idx = index.take(indexer[~mask])
+
+        # left/right/outer (if outer is a subset of left/right) join
+        reset_dtype = index.empty
+        if idx is None:
+
+            # try to preserve index, worse case we reset it
+            if index.shape == mask.shape:
+                idx = np.where(mask, index, index.take(indexer))
+            else:
+                if mask.any():
+                    return np.arange(len(mask))
+                else:
+                    idx = index.take(indexer)
+
+        if isinstance(index, MultiIndex):
+            if len(idx) == 0:
+                return MultiIndex.from_arrays(np.empty((index.nlevels, 0)), names=names)
+            else:
+                return MultiIndex.from_tuples(idx, names=names)
+        else:
+
+            # a bit of spagetti soup
+            create_categorial = False
+            if infer_index_type and issubclass(type(index), Index):
+
+                # TODO: Int64Index doesn't like Int64Index(np.ndarray) nor Categorical
+                if is_integer_dtype(index):
+                    index_type = Index
+                elif is_categorical_dtype(index):
+                    create_categorial = True
+                    index_type = type(index)
+                    if is_categorical_dtype(other_index):
+                        union_cat = index.categories.union(other_index.categories)
+                    else:
+                        union_cat = index.categories.union(other_index)
+                else:
+                    index_type = type(index)
+
+            else:
+                if is_categorical_dtype(other_index):
+                    create_categorial = True
+                    index_type = type(other_index)
+                    if is_categorical_dtype(index):
+                        union_cat = other_index.categories.union(index.categories)
+                    else:
+                        union_cat = other_index.categories.union(index)
+                else:
+                    index_type = Index
+
+            if create_categorial:
+                return index_type(idx, categories=union_cat, name=names[0])
+            elif reset_dtype:
+                return index_type(idx, name=names[0])
+            elif convert_type is None:
+                if is_period_dtype(index.dtype) or is_datetime64tz_dtype(index.dtype):
+                    return index_type(idx, name=names[0])
+                else:
+                    return index_type(idx, dtype=index.dtype, name=names[0])
+            else:
+                if is_period_dtype(other_index.dtype) or is_datetime64tz_dtype(
+                    other_index.dtype
+                ):
+                    return index_type(idx, name=names[0])
+                else:
+                    return index_type(idx, dtype=convert_type, name=names[0])
 
     def _get_merge_keys(self):
         """