ERR: ValueError when merging on incompatible dtypes

Author: reidy-p

doc/source/whatsnew/v0.22.0.txt

@@ -186,6 +186,8 @@ Other API Changes
 - :func:`Series.fillna` now raises a ``TypeError`` instead of a ``ValueError`` when passed a list, tuple or DataFrame as a ``value`` (:issue:`18293`)
 - :func:`pandas.DataFrame.merge` no longer casts a ``float`` column to ``object`` when merging on ``int`` and ``float`` columns (:issue:`16572`)
 - The default NA value for :class:`UInt64Index` has changed from 0 to ``NaN``, which impacts methods that mask with NA, such as ``UInt64Index.where()`` (:issue:`18398`)
+- Building pandas for development now requires ``cython >= 0.24`` (:issue:`18613`)
+- :func:`pandas.merge` now raises a ``ValueError`` when trying to merge on incompatible data types (:issue:`9780`)
 
 .. _whatsnew_0220.deprecations:
 

pandas/core/reshape/merge.py

@@ -27,6 +27,7 @@
     is_dtype_equal,
     is_bool,
     is_list_like,
+    is_datetimelike,
     _ensure_int64,
     _ensure_float64,
     _ensure_object,
@@ -962,14 +963,30 @@ def _maybe_coerce_merge_keys(self):
                 elif lib.infer_dtype(lk) == lib.infer_dtype(rk):
                     pass
 
-            # Houston, we have a problem!
-            # let's coerce to object if the dtypes aren't
-            # categorical, otherwise coerce to the category
-            # dtype. If we coerced categories to object,
-            # then we would lose type information on some
-            # columns, and end up trying to merge
-            # incompatible dtypes. See GH 16900.
             else:
+
+                # Check if we are trying to merge on obviously
+                # incompatible dtypes GH 9780
+                msg = ("You are trying to merge on {lk_dtype} and "
+                       "{rk_dtype} columns. If you wish to proceed "
+                       "you should use pd.concat".format(lk_dtype=lk.dtype,
+                                                         rk_dtype=rk.dtype))
+                if is_numeric_dtype(lk) and not is_numeric_dtype(rk):
+                    raise ValueError(msg)
+                elif not is_numeric_dtype(lk) and is_numeric_dtype(rk):
+                    raise ValueError(msg)
+                elif is_datetimelike(lk) and not is_datetimelike(rk):
+                    raise ValueError(msg)
+                elif not is_datetimelike(lk) and is_datetimelike(rk):
+                    raise ValueError(msg)
+
+                # Houston, we have a problem!
+                # let's coerce to object if the dtypes aren't
+                # categorical, otherwise coerce to the category
+                # dtype. If we coerced categories to object,
+                # then we would lose type information on some
+                # columns, and end up trying to merge
+                # incompatible dtypes. See GH 16900.
                 if name in self.left.columns:
                     typ = lk.categories.dtype if lk_is_cat else object
                     self.left = self.left.assign(

pandas/tests/reshape/merge/test_merge.py

@@ -6,6 +6,7 @@
 from numpy import nan
 import numpy as np
 import random
+import re
 
 import pandas as pd
 from pandas.compat import lrange, lzip
@@ -1385,14 +1386,27 @@ class TestMergeDtypes(object):
 
     def test_different(self, df):
 
-        # we expect differences by kind
-        # to be ok, while other differences should return object
-
         left = df
         for col in df.columns:
             right = DataFrame({'A': df[col]})
-            result = pd.merge(left, right, on='A')
-            assert is_object_dtype(result.A.dtype)
+            # GH 9780
+            # We allow merging on object and categorical cols and cast 
+            # categorical cols to object
+            if (is_categorical_dtype(right['A'].dtype) or
+               is_object_dtype(right['A'].dtype)):
+                result = pd.merge(left, right, on='A')
+                assert is_object_dtype(result.A.dtype)
+            # GH 9780
+            # We raise for merging on object col and int/float col and
+            # merging on categorical col and int/float col
+            else:
+                msg = ("You are trying to merge on "
+                       "{lk_dtype} and {rk_dtype} columns. "
+                       "If you wish to proceed you should use "
+                       "pd.concat".format(lk_dtype=left['A'].dtype,
+                                          rk_dtype=right['A'].dtype))
+                with tm.assert_raises_regex(ValueError, msg):
+                    pd.merge(left, right, on='A')
 
     @pytest.mark.parametrize('d1', [np.int64, np.int32,
                                     np.int16, np.int8, np.uint8])
@@ -1462,6 +1476,42 @@ def test_merge_on_ints_floats_warning(self):
             result = B.merge(A, left_on='Y', right_on='X')
             assert_frame_equal(result, expected[['Y', 'X']])
 
+    @pytest.mark.parametrize('df1_vals, df2_vals', [
+        ([0, 1, 2], ["0", "1", "2"]),
+        ([0.0, 1.0, 2.0], ["0", "1", "2"]),
+        ([0, 1, 2], [u"0", u"1", u"2"]),
+        (pd.date_range('1/1/2011', periods=2, freq='D'), ['2011-01-01',
+                                                          '2011-01-02']),
+        (pd.date_range('1/1/2011', periods=2, freq='D'), [0, 1]),
+        (pd.date_range('1/1/2011', periods=2, freq='D'), [0.0, 1.0]),
+        ([0, 1, 2], Series(['a', 'b', 'a']).astype('category')),
+        ([0.0, 1.0, 2.0], Series(['a', 'b', 'a']).astype('category')),
+    ])
+    def test_merge_incompat_dtypes(self, df1_vals, df2_vals):
+        # GH 9780
+        # Raise a ValueError when a user tries to merge on
+        # dtypes that are incompatible (e.g., obj and int/float)
+
+        df1 = DataFrame({'A': df1_vals})
+        df2 = DataFrame({'A': df2_vals})
+
+        msg = ("You are trying to merge on {lk_dtype} and "
+               "{rk_dtype} columns. If you wish to proceed "
+               "you should use pd.concat".format(lk_dtype=df1['A'].dtype,
+                                                 rk_dtype=df2['A'].dtype))
+        msg = re.escape(msg)
+        with tm.assert_raises_regex(ValueError, msg):
+            pd.merge(df1, df2, on=['A'])
+
+        # Check that error still raised when swapping order of dataframes
+        msg = ("You are trying to merge on {lk_dtype} and "
+               "{rk_dtype} columns. If you wish to proceed "
+               "you should use pd.concat".format(lk_dtype=df2['A'].dtype,
+                                                 rk_dtype=df1['A'].dtype))
+        msg = re.escape(msg)
+        with tm.assert_raises_regex(ValueError, msg):
+            pd.merge(df2, df1, on=['A'])
+
 
 @pytest.fixture
 def left():