ENH - Index set operation modifications to address issue #23525 (#23538)

Author: ms7463

doc/source/whatsnew/v0.25.0.rst

@@ -154,6 +154,33 @@ returned if all the columns were dummy encoded, and a :class:`DataFrame` otherwi
 Providing any ``SparseSeries`` or ``SparseDataFrame`` to :func:`concat` will
 cause a ``SparseSeries`` or ``SparseDataFrame`` to be returned, as before.
 
+.. _whatsnew_0250.api_breaking.incompatible_index_unions
+
+Incompatible Index Type Unions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When performing :func:`Index.union` operations between objects of incompatible dtypes,
+the result will be a base :class:`Index` of dtype ``object``. This behavior holds true for
+unions between :class:`Index` objects that previously would have been prohibited. The dtype
+of empty :class:`Index` objects will now be evaluated before performing union operations
+rather than simply returning the other :class:`Index` object. :func:`Index.union` can now be
+considered commutative, such that ``A.union(B) == B.union(A)`` (:issue:`23525`).
+
+*Previous Behavior*:
+
+    In [1]: pd.period_range('19910905', periods=2).union(pd.Int64Index([1, 2, 3]))
+    ...
+    ValueError: can only call with other PeriodIndex-ed objects
+
+    In [2]: pd.Index([], dtype=object).union(pd.Index([1, 2, 3]))
+    Out[2]: Int64Index([1, 2, 3], dtype='int64')
+
+*New Behavior*:
+
+.. ipython:: python
+
+    pd.period_range('19910905', periods=2).union(pd.Int64Index([1, 2, 3]))
+    pd.Index([], dtype=object).union(pd.Index([1, 2, 3]))
 
 ``DataFrame`` groupby ffill/bfill no longer return group labels
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

pandas/core/indexes/base.py

@@ -20,11 +20,10 @@
     ensure_categorical, ensure_int64, ensure_object, ensure_platform_int,
     is_bool, is_bool_dtype, is_categorical, is_categorical_dtype,
     is_datetime64_any_dtype, is_datetime64tz_dtype, is_dtype_equal,
-    is_dtype_union_equal, is_extension_array_dtype, is_float, is_float_dtype,
-    is_hashable, is_integer, is_integer_dtype, is_interval_dtype, is_iterator,
-    is_list_like, is_object_dtype, is_period_dtype, is_scalar,
-    is_signed_integer_dtype, is_timedelta64_dtype, is_unsigned_integer_dtype,
-    pandas_dtype)
+    is_extension_array_dtype, is_float, is_float_dtype, is_hashable,
+    is_integer, is_integer_dtype, is_interval_dtype, is_iterator, is_list_like,
+    is_object_dtype, is_period_dtype, is_scalar, is_signed_integer_dtype,
+    is_timedelta64_dtype, is_unsigned_integer_dtype, pandas_dtype)
 import pandas.core.dtypes.concat as _concat
 from pandas.core.dtypes.generic import (
     ABCDataFrame, ABCDateOffset, ABCDatetimeArray, ABCIndexClass,
@@ -2262,6 +2261,47 @@ def _get_reconciled_name_object(self, other):
             return self._shallow_copy(name=name)
         return self
 
+    def _union_incompatible_dtypes(self, other, sort):
+        """
+        Casts this and other index to object dtype to allow the formation
+        of a union between incompatible types.
+
+        Parameters
+        ----------
+        other : Index or array-like
+        sort : False or None, default False
+            Whether to sort the resulting index.
+
+            * False : do not sort the result.
+            * None : sort the result, except when `self` and `other` are equal
+              or when the values cannot be compared.
+
+        Returns
+        -------
+        Index
+        """
+        this = self.astype(object, copy=False)
+        # cast to Index for when `other` is list-like
+        other = Index(other).astype(object, copy=False)
+        return Index.union(this, other, sort=sort).astype(object, copy=False)
+
+    def _is_compatible_with_other(self, other):
+        """
+        Check whether this and the other dtype are compatible with each other.
+        Meaning a union can be formed between them without needing to be cast
+        to dtype object.
+
+        Parameters
+        ----------
+        other : Index or array-like
+
+        Returns
+        -------
+        bool
+        """
+        return (type(self) is type(other)
+                and is_dtype_equal(self.dtype, other.dtype))
+
     def _validate_sort_keyword(self, sort):
         if sort not in [None, False]:
             raise ValueError("The 'sort' keyword only takes the values of "
@@ -2271,6 +2311,11 @@ def union(self, other, sort=None):
         """
         Form the union of two Index objects.
 
+        If the Index objects are incompatible, both Index objects will be
+        cast to dtype('object') first.
+
+            .. versionchanged:: 0.25.0
+
         Parameters
         ----------
         other : Index or array-like
@@ -2300,30 +2345,54 @@ def union(self, other, sort=None):
         Examples
         --------
 
+        Union matching dtypes
+
         >>> idx1 = pd.Index([1, 2, 3, 4])
         >>> idx2 = pd.Index([3, 4, 5, 6])
         >>> idx1.union(idx2)
         Int64Index([1, 2, 3, 4, 5, 6], dtype='int64')
+
+        Union mismatched dtypes
+
+        >>> idx1 = pd.Index(['a', 'b', 'c', 'd'])
+        >>> idx2 = pd.Index([1, 2, 3, 4])
+        >>> idx1.union(idx2)
+        Index(['a', 'b', 'c', 'd', 1, 2, 3, 4], dtype='object')
         """
         self._validate_sort_keyword(sort)
         self._assert_can_do_setop(other)
-        other = ensure_index(other)
 
-        if len(other) == 0 or self.equals(other):
+        if not self._is_compatible_with_other(other):
+            return self._union_incompatible_dtypes(other, sort=sort)
+
+        return self._union(other, sort=sort)
+
+    def _union(self, other, sort):
+        """
+        Specific union logic should go here. In subclasses, union behavior
+        should be overwritten here rather than in `self.union`.
+
+        Parameters
+        ----------
+        other : Index or array-like
+        sort : False or None, default False
+            Whether to sort the resulting index.
+
+            * False : do not sort the result.
+            * None : sort the result, except when `self` and `other` are equal
+              or when the values cannot be compared.
+
+        Returns
+        -------
+        Index
+        """
+
+        if not len(other) or self.equals(other):
             return self._get_reconciled_name_object(other)
 
-        if len(self) == 0:
+        if not len(self):
             return other._get_reconciled_name_object(self)
 
-        # TODO: is_dtype_union_equal is a hack around
-        # 1. buggy set ops with duplicates (GH #13432)
-        # 2. CategoricalIndex lacking setops (GH #10186)
-        # Once those are fixed, this workaround can be removed
-        if not is_dtype_union_equal(self.dtype, other.dtype):
-            this = self.astype('O')
-            other = other.astype('O')
-            return this.union(other, sort=sort)
-
         # TODO(EA): setops-refactor, clean all this up
         if is_period_dtype(self) or is_datetime64tz_dtype(self):
             lvals = self._ndarray_values
@@ -2370,6 +2439,7 @@ def union(self, other, sort=None):
     def _wrap_setop_result(self, other, result):
         return self._constructor(result, name=get_op_result_name(self, other))
 
+    # TODO: standardize return type of non-union setops type(self vs other)
     def intersection(self, other, sort=False):
         """
         Form the intersection of two Index objects.

pandas/core/indexes/datetimes.py

@@ -451,35 +451,9 @@ def _formatter_func(self):
     # --------------------------------------------------------------------
     # Set Operation Methods
 
-    def union(self, other, sort=None):
-        """
-        Specialized union for DatetimeIndex objects. If combine
-        overlapping ranges with the same DateOffset, will be much
-        faster than Index.union
-
-        Parameters
-        ----------
-        other : DatetimeIndex or array-like
-        sort : bool or None, default None
-            Whether to sort the resulting Index.
-
-            * None : Sort the result, except when
-
-              1. `self` and `other` are equal.
-              2. `self` or `other` has length 0.
-              3. Some values in `self` or `other` cannot be compared.
-                 A RuntimeWarning is issued in this case.
-
-            * False : do not sort the result
-
-            .. versionadded:: 0.25.0
-
-        Returns
-        -------
-        y : Index or DatetimeIndex
-        """
-        self._validate_sort_keyword(sort)
-        self._assert_can_do_setop(other)
+    def _union(self, other, sort):
+        if not len(other) or self.equals(other) or not len(self):
+            return super()._union(other, sort=sort)
 
         if len(other) == 0 or self.equals(other) or len(self) == 0:
             return super().union(other, sort=sort)
@@ -495,7 +469,7 @@ def union(self, other, sort=None):
         if this._can_fast_union(other):
             return this._fast_union(other, sort=sort)
         else:
-            result = Index.union(this, other, sort=sort)
+            result = Index._union(this, other, sort=sort)
             if isinstance(result, DatetimeIndex):
                 # TODO: we shouldn't be setting attributes like this;
                 #  in all the tests this equality already holds

pandas/core/indexes/interval.py

@@ -964,19 +964,6 @@ def insert(self, loc, item):
         new_right = self.right.insert(loc, right_insert)
         return self._shallow_copy(new_left, new_right)
 
-    def _as_like_interval_index(self, other):
-        self._assert_can_do_setop(other)
-        other = ensure_index(other)
-        if not isinstance(other, IntervalIndex):
-            msg = ('the other index needs to be an IntervalIndex too, but '
-                   'was type {}').format(other.__class__.__name__)
-            raise TypeError(msg)
-        elif self.closed != other.closed:
-            msg = ('can only do set operations between two IntervalIndex '
-                   'objects that are closed on the same side')
-            raise ValueError(msg)
-        return other
-
     def _concat_same_dtype(self, to_concat, name):
         """
         assert that we all have the same .closed
@@ -1092,7 +1079,17 @@ def overlaps(self, other):
 
     def _setop(op_name, sort=None):
         def func(self, other, sort=sort):
-            other = self._as_like_interval_index(other)
+            self._assert_can_do_setop(other)
+            other = ensure_index(other)
+            if not isinstance(other, IntervalIndex):
+                result = getattr(self.astype(object), op_name)(other)
+                if op_name in ('difference',):
+                    result = result.astype(self.dtype)
+                return result
+            elif self.closed != other.closed:
+                msg = ('can only do set operations between two IntervalIndex '
+                       'objects that are closed on the same side')
+                raise ValueError(msg)
 
             # GH 19016: ensure set op will not return a prohibited dtype
             subtypes = [self.dtype.subtype, other.dtype.subtype]
@@ -1114,6 +1111,7 @@ def func(self, other, sort=sort):
 
             return type(self).from_tuples(result, closed=self.closed,
                                           name=result_name)
+
         return func
 
     @property

pandas/core/indexes/numeric.py

@@ -9,6 +9,7 @@
     is_bool, is_bool_dtype, is_dtype_equal, is_extension_array_dtype, is_float,
     is_integer_dtype, is_scalar, needs_i8_conversion, pandas_dtype)
 import pandas.core.dtypes.concat as _concat
+from pandas.core.dtypes.generic import ABCInt64Index, ABCRangeIndex
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import algorithms
@@ -221,6 +222,13 @@ def _assert_safe_casting(cls, data, subarr):
                 raise TypeError('Unsafe NumPy casting, you must '
                                 'explicitly cast')
 
+    def _is_compatible_with_other(self, other):
+        return (
+            super()._is_compatible_with_other(other)
+            or all(isinstance(type(obj), (ABCInt64Index, ABCRangeIndex))
+                   for obj in [self, other])
+        )
+
 
 Int64Index._add_numeric_methods()
 Int64Index._add_logical_methods()

pandas/core/indexes/period.py

@@ -791,6 +791,11 @@ def join(self, other, how='left', level=None, return_indexers=False,
         """
         self._assert_can_do_setop(other)
 
+        if not isinstance(other, PeriodIndex):
+            return self.astype(object).join(other, how=how, level=level,
+                                            return_indexers=return_indexers,
+                                            sort=sort)
+
         result = Int64Index.join(self, other, how=how, level=level,
                                  return_indexers=return_indexers,
                                  sort=sort)
@@ -807,10 +812,9 @@ def intersection(self, other, sort=False):
     def _assert_can_do_setop(self, other):
         super()._assert_can_do_setop(other)
 
-        if not isinstance(other, PeriodIndex):
-            raise ValueError('can only call with other PeriodIndex-ed objects')
-
-        if self.freq != other.freq:
+        # *Can't* use PeriodIndexes of different freqs
+        # *Can* use PeriodIndex/DatetimeIndex
+        if isinstance(other, PeriodIndex) and self.freq != other.freq:
             msg = DIFFERENT_FREQ.format(cls=type(self).__name__,
                                         own_freq=self.freqstr,
                                         other_freq=other.freqstr)

pandas/core/indexes/range.py

@@ -470,7 +470,7 @@ def _extended_gcd(self, a, b):
             old_t, t = t, old_t - quotient * t
         return old_r, old_s, old_t
 
-    def union(self, other, sort=None):
+    def _union(self, other, sort):
         """
         Form the union of two Index objects and sorts if possible
 
@@ -490,9 +490,8 @@ def union(self, other, sort=None):
         -------
         union : Index
         """
-        self._assert_can_do_setop(other)
-        if len(other) == 0 or self.equals(other) or len(self) == 0:
-            return super().union(other, sort=sort)
+        if not len(other) or self.equals(other) or not len(self):
+            return super()._union(other, sort=sort)
 
         if isinstance(other, RangeIndex) and sort is None:
             start_s, step_s = self._start, self._step
@@ -530,8 +529,7 @@ def union(self, other, sort=None):
                         (start_s + step_o >= start_o) and
                         (end_s - step_o <= end_o)):
                     return RangeIndex(start_r, end_r + step_o, step_o)
-
-        return self._int64index.union(other, sort=sort)
+        return self._int64index._union(other, sort=sort)
 
     @Appender(_index_shared_docs['join'])
     def join(self, other, how='left', level=None, return_indexers=False,

pandas/core/indexes/timedeltas.py

@@ -329,24 +329,9 @@ def astype(self, dtype, copy=True):
             return Index(result.astype('i8'), name=self.name)
         return DatetimeIndexOpsMixin.astype(self, dtype, copy=copy)
 
-    def union(self, other):
-        """
-        Specialized union for TimedeltaIndex objects. If combine
-        overlapping ranges with the same DateOffset, will be much
-        faster than Index.union
-
-        Parameters
-        ----------
-        other : TimedeltaIndex or array-like
-
-        Returns
-        -------
-        y : Index or TimedeltaIndex
-        """
-        self._assert_can_do_setop(other)
-
+    def _union(self, other, sort):
         if len(other) == 0 or self.equals(other) or len(self) == 0:
-            return super().union(other)
+            return super()._union(other, sort=sort)
 
         if not isinstance(other, TimedeltaIndex):
             try:
@@ -358,7 +343,7 @@ def union(self, other):
         if this._can_fast_union(other):
             return this._fast_union(other)
         else:
-            result = Index.union(this, other)
+            result = Index._union(this, other, sort=sort)
             if isinstance(result, TimedeltaIndex):
                 if result.freq is None:
                     result.freq = to_offset(result.inferred_freq)