BUG: Concat multiple different ExtensionArray types

doc/source/whatsnew/v0.24.0.txt

@@ -492,6 +492,15 @@ Previous Behavior:
 ExtensionType Changes
 ^^^^^^^^^^^^^^^^^^^^^
 
+**:class:`pandas.api.extensions.ExtensionDtype` Equality and Hashability**
+
+Pandas now requires that extension dtypes be hashable. The base class implements
+a default ``__eq__`` and ``__hash__``. If you have a parametrized dtype, you should
+update the ``ExtensionDtype._metadata`` tuple to match the signature of your
+``__init__`` method. See :class:`pandas.api.extensions.ExtensionDtype` for more (:issue:`22476`).
+
+**Other changes**
+
 - ``ExtensionArray`` has gained the abstract methods ``.dropna()`` (:issue:`21185`)
 - ``ExtensionDtype`` has gained the ability to instantiate from string dtypes, e.g. ``decimal`` would instantiate a registered ``DecimalDtype``; furthermore
   the ``ExtensionDtype`` has gained the method ``construct_array_type`` (:issue:`21185`)
@@ -505,6 +514,7 @@ ExtensionType Changes
 - :meth:`Series.astype` and :meth:`DataFrame.astype` now dispatch to :meth:`ExtensionArray.astype` (:issue:`21185:`).
 - Slicing a single row of a ``DataFrame`` with multiple ExtensionArrays of the same type now preserves the dtype, rather than coercing to object (:issue:`22784`)
 - Added :meth:`pandas.api.types.register_extension_dtype` to register an extension type with pandas (:issue:`22664`)
+- Bug in concatenation an Series with two different extension dtypes not casting to object dtype (:issue:`22994`)
 - Updated the ``.type`` attribute for ``PeriodDtype``, ``DatetimeTZDtype``, and ``IntervalDtype`` to be instances of the dtype (``Period``, ``Timestamp``, and ``Interval`` respectively) (:issue:`22938`)
 
 .. _whatsnew_0240.api.incompatibilities:

pandas/core/dtypes/base.py

@@ -22,14 +22,17 @@ class _DtypeOpsMixin(object):
     # of the NA value, not the physical NA vaalue for storage.
     # e.g. for JSONArray, this is an empty dictionary.
     na_value = np.nan
+    _metadata = ()
 
     def __eq__(self, other):
         """Check whether 'other' is equal to self.
 
-        By default, 'other' is considered equal if
+        By default, 'other' is considered equal if either
 
         * it's a string matching 'self.name'.
-        * it's an instance of this type.
+        * it's an instance of this type and all of the
+          the attributes in ``self._metadata`` are equal between
+          `self` and `other`.
 
         Parameters
         ----------
@@ -40,11 +43,19 @@ def __eq__(self, other):
         bool
         """
         if isinstance(other, compat.string_types):
-            return other == self.name
-        elif isinstance(other, type(self)):
-            return True
-        else:
-            return False
+            try:
+                other = self.construct_from_string(other)
+            except TypeError:
+                return False
+        if isinstance(other, type(self)):
+            return all(
+                getattr(self, attr) == getattr(other, attr)
+                for attr in self._metadata
+            )
+        return False
+
+    def __hash__(self):
+        return hash(tuple(getattr(self, attr) for attr in self._metadata))
 
     def __ne__(self, other):
         return not self.__eq__(other)
@@ -161,6 +172,26 @@ class ExtensionDtype(_DtypeOpsMixin):
     The `na_value` class attribute can be used to set the default NA value
     for this type. :attr:`numpy.nan` is used by default.
 
+    ExtensionDtypes are required to be hashable. The base class provides
+    a default implementation, which relies on the ``_metadata`` class
+    attribute. ``_metadata`` should be a tuple containing the strings
+    that define your data type. For example, with ``PeriodDtype`` that's
+    the ``freq`` attribute.
+
+    **If you have a parametrized dtype you should set the ``_metadata``
+    class property**.
+
+    Ideally, the attributes in ``_metadata`` will match the
+    parameters to your ``ExtensionDtype.__init__`` (if any). If any of
+    the attributes in ``_metadata`` don't implement the standard
+    ``__eq__`` or ``__hash__``, the default implementations here will not
+    work.
+
+    .. versionchanged:: 0.24.0
+
+       Added ``_metadata``, ``__hash__``, and changed the default definition
+       of ``__eq__``.
+
     This class does not inherit from 'abc.ABCMeta' for performance reasons.
     Methods and properties required by the interface raise
     ``pandas.errors.AbstractMethodError`` and no ``register`` method is

pandas/core/dtypes/dtypes.py

@@ -101,7 +101,6 @@ class PandasExtensionDtype(_DtypeOpsMixin):
     base = None
     isbuiltin = 0
     isnative = 0
-    _metadata = []
     _cache = {}
 
     def __unicode__(self):
@@ -209,7 +208,7 @@ class CategoricalDtype(PandasExtensionDtype, ExtensionDtype):
     kind = 'O'
     str = '|O08'
     base = np.dtype('O')
-    _metadata = ['categories', 'ordered']
+    _metadata = ('categories', 'ordered')
     _cache = {}
 
     def __init__(self, categories=None, ordered=None):
@@ -485,7 +484,7 @@ class DatetimeTZDtype(PandasExtensionDtype):
     str = '|M8[ns]'
     num = 101
     base = np.dtype('M8[ns]')
-    _metadata = ['unit', 'tz']
+    _metadata = ('unit', 'tz')
     _match = re.compile(r"(datetime64|M8)\[(?P<unit>.+), (?P<tz>.+)\]")
     _cache = {}
 
@@ -589,7 +588,7 @@ class PeriodDtype(PandasExtensionDtype):
     str = '|O08'
     base = np.dtype('O')
     num = 102
-    _metadata = ['freq']
+    _metadata = ('freq',)
     _match = re.compile(r"(P|p)eriod\[(?P<freq>.+)\]")
     _cache = {}
 
@@ -709,7 +708,7 @@ class IntervalDtype(PandasExtensionDtype, ExtensionDtype):
     str = '|O08'
     base = np.dtype('O')
     num = 103
-    _metadata = ['subtype']
+    _metadata = ('subtype',)
     _match = re.compile(r"(I|i)nterval\[(?P<subtype>.+)\]")
     _cache = {}
 

pandas/core/internals/managers.py

@@ -1634,8 +1634,7 @@ def concat(self, to_concat, new_axis):
         # check if all series are of the same block type:
         if len(non_empties) > 0:
             blocks = [obj.blocks[0] for obj in non_empties]
-
-            if all(type(b) is type(blocks[0]) for b in blocks[1:]):  # noqa
+            if len({b.dtype for b in blocks}) == 1:
                 new_block = blocks[0].concat_same_type(blocks)
             else:
                 values = [x.values for x in blocks]

pandas/tests/extension/base/dtype.py

@@ -49,6 +49,10 @@ def test_eq_with_str(self, dtype):
     def test_eq_with_numpy_object(self, dtype):
         assert dtype != np.dtype('object')
 
+    def test_eq_with_self(self, dtype):
+        assert dtype == dtype
+        assert dtype != object()
+
     def test_array_type(self, data, dtype):
         assert dtype.construct_array_type() is type(data)
 
@@ -81,3 +85,6 @@ def test_check_dtype(self, data):
                              index=list('ABCD'))
         result = df.dtypes.apply(str) == str(dtype)
         self.assert_series_equal(result, expected)
+
+    def test_hashable(self, dtype):
+        hash(dtype)  # no error

pandas/tests/extension/decimal/array.py

@@ -15,15 +15,11 @@ class DecimalDtype(ExtensionDtype):
     type = decimal.Decimal
     name = 'decimal'
     na_value = decimal.Decimal('NaN')
+    _metadata = ('context',)
 
     def __init__(self, context=None):
         self.context = context or decimal.getcontext()
 
-    def __eq__(self, other):
-        if isinstance(other, type(self)):
-            return self.context == other.context
-        return super(DecimalDtype, self).__eq__(other)
-
     def __repr__(self):
         return 'DecimalDtype(context={})'.format(self.context)
 

pandas/tests/extension/json/array.py

@@ -27,6 +27,7 @@
 class JSONDtype(ExtensionDtype):
     type = compat.Mapping
     name = 'json'
+
     try:
         na_value = collections.UserDict()
     except AttributeError:

pandas/tests/reshape/test_concat.py

@@ -1,6 +1,7 @@
 from warnings import catch_warnings, simplefilter
 from itertools import combinations
 from collections import deque
+from decimal import Decimal
 
 import datetime as dt
 import dateutil
@@ -19,6 +20,7 @@
 from pandas.util import testing as tm
 from pandas.util.testing import (assert_frame_equal,
                                  makeCustomDataframe as mkdf)
+from pandas.tests.extension.decimal import to_decimal
 
 import pytest
 
@@ -2361,6 +2363,17 @@ def test_concat_datetime_timezone(self):
                                 index=idx1.append(idx1))
         tm.assert_frame_equal(result, expected)
 
+    def test_concat_different_extension_dtypes_upcasts(self):
+        a = pd.Series(pd.core.arrays.integer_array([1, 2]))
+        b = pd.Series(to_decimal([1, 2]))
+
+        result = pd.concat([a, b], ignore_index=True)
+        expected = pd.Series([
+            1, 2,
+            Decimal(1), Decimal(2)
+        ], dtype=object)
+        tm.assert_series_equal(result, expected)
+
 
 @pytest.mark.parametrize('pdt', [pd.Series, pd.DataFrame, pd.Panel])
 @pytest.mark.parametrize('dt', np.sctypes['float'])