BUG: DataFrame mixed-type arithmetic column-wise, fix DataFrame.diff upcasting->object bug close #1896

Author: wesm

RELEASE.rst

@@ -59,6 +59,8 @@ pandas 0.9.0
 
 **Bug fixes**
 
+  - Perform arithmetic column-by-column in mixed-type DataFrame to avoid type
+    upcasting issues. Caused downstream DataFrame.diff bug (#1896)
   - Fix matplotlib auto-color assignment when no custom spectrum passed. Also
     respect passed color keyword argument (#1711)
   - Fix resampling logical error with closed='left' (#1726)

pandas/core/frame.py

@@ -3139,23 +3139,30 @@ def _combine_frame(self, other, func, fill_value=None, level=None):
         this, other = self.align(other, join='outer', level=level, copy=False)
         new_index, new_columns = this.index, this.columns
 
-        this_vals = this.values
-        other_vals = other.values
+        def _arith_op(left, right):
+            if fill_value is not None:
+                left_mask = isnull(left)
+                right_mask = isnull(right)
+                left = left.copy()
+                right = right.copy()
+
+                # one but not both
+                mask = left_mask ^ right_mask
+                left[left_mask & mask] = fill_value
+                right[right_mask & mask] = fill_value
+
+            return func(left, right)
+
+        if this._is_mixed_type or other._is_mixed_type:
+            # XXX no good for duplicate columns
+            result = {}
+            for col in this:
+                result[col] = func(this[col].values, other[col].values)
+        else:
+            result = _arith_op(this.values, other.values)
 
-        if fill_value is not None:
-            this_mask = isnull(this_vals)
-            other_mask = isnull(other_vals)
-            this_vals = this_vals.copy()
-            other_vals = other_vals.copy()
-
-            # one but not both
-            mask = this_mask ^ other_mask
-            this_vals[this_mask & mask] = fill_value
-            other_vals[other_mask & mask] = fill_value
-
-        result = func(this_vals, other_vals)
-        return self._constructor(result, index=new_index, columns=new_columns,
-                                 copy=False)
+        return self._constructor(result, index=new_index,
+                                 columns=new_columns, copy=False)
 
     def _indexed_same(self, other):
         same_index = self.index.equals(other.index)

pandas/tests/test_frame.py

@@ -2710,7 +2710,7 @@ def test_operators(self):
 
     def test_operators_none_as_na(self):
         df = DataFrame({"col1": [2,5.0,123,None],
-                        "col2": [1,2,3,4]})
+                        "col2": [1,2,3,4]}, dtype=object)
 
         ops = [operator.add, operator.sub, operator.mul, operator.truediv]
 
@@ -3553,7 +3553,7 @@ def test_to_excel_float_format(self):
             assert_frame_equal(rs, xp)
             os.remove(filename)
 
-    def test_000to_excel_unicode_filename(self):
+    def test_to_excel_unicode_filename(self):
         try:
             import xlwt
             import openpyxl
@@ -4906,6 +4906,13 @@ def test_diff(self):
         assert_series_equal(the_diff['A'],
                             self.tsframe['A'] - self.tsframe['A'].shift(1))
 
+    def test_diff_mixed_dtype(self):
+        df = DataFrame(np.random.randn(5, 3))
+        df['A'] = np.array([1, 2, 3, 4, 5], dtype=object)
+
+        result = df.diff()
+        self.assert_(result[0].dtype == np.float64)
+
     def test_pct_change(self):
         rs = self.tsframe.pct_change(fill_method=None)
         assert_frame_equal(rs, self.tsframe / self.tsframe.shift(1) - 1)