BUG: improper handling of mis-matched columns in DataFrame.combine, GH #529

Author: wesm

RELEASE.rst

@@ -81,6 +81,7 @@ pandas 0.6.2
   - Handle rows skipped at beginning of file in read_* functions (GH #505)
   - Handle improper dtype casting in ``set_value`` methods
   - Unary '-' / __neg__ operator on DataFrame was returning integer values
+  - Unbox 0-dim ndarrays from certain operators like all, any in Series
 
 Thanks
 ------

pandas/core/frame.py

@@ -1999,7 +1999,7 @@ def combine(self, other, func, fill_value=None):
         if not self:
             return other.copy()
 
-        this, other = self.align(other, axis=0, copy=False)
+        this, other = self.align(other, copy=False)
         new_index = this.index
 
         # sorts if possible
@@ -2008,30 +2008,24 @@ def combine(self, other, func, fill_value=None):
 
         result = {}
         for col in new_columns:
-            if col in this and col in other:
-                series = this[col].values
-                otherSeries = other[col].values
-
-                if do_fill:
-                    this_mask = isnull(series)
-                    other_mask = isnull(otherSeries)
-                    series = series.copy()
-                    otherSeries = otherSeries.copy()
-                    series[this_mask] = fill_value
-                    otherSeries[other_mask] = fill_value
-
-                arr = func(series, otherSeries)
-
-                if do_fill:
-                    arr = com.ensure_float(arr)
-                    arr[this_mask & other_mask] = nan
-
-                result[col] = arr
-
-            elif col in this:
-                result[col] = this[col]
-            elif col in other:
-                result[col] = other[col]
+            series = this[col].values
+            otherSeries = other[col].values
+
+            if do_fill:
+                this_mask = isnull(series)
+                other_mask = isnull(otherSeries)
+                series = series.copy()
+                otherSeries = otherSeries.copy()
+                series[this_mask] = fill_value
+                otherSeries[other_mask] = fill_value
+
+            arr = func(series, otherSeries)
+
+            if do_fill:
+                arr = com.ensure_float(arr)
+                arr[this_mask & other_mask] = nan
+
+            result[col] = arr
 
         return self._constructor(result, index=new_index, columns=new_columns)
 

pandas/core/series.py

@@ -88,6 +88,16 @@ def f(self, other, fill_value=None):
     f.__name__ = name
     return f
 
+def _unbox(func):
+    def f(self, *args, **kwargs):
+        result = func(self, *args, **kwargs)
+        if isinstance(result, np.ndarray) and result.ndim == 0:
+            return result.item()
+        return result
+    f.__doc__ = func.__doc__
+    f.__name__ = func.__name__
+    return f
+
 _stat_doc = """
 Return %(name)s  of values
 %(na_action)s
@@ -566,6 +576,12 @@ def iteritems(self):
         __rdiv__ = _arith_method(lambda x, y: y / x, '__div__')
         __idiv__ = __div__
 
+    #----------------------------------------------------------------------
+    # unbox reductions
+
+    all = _unbox(np.ndarray.all)
+    any = _unbox(np.ndarray.any)
+
     #----------------------------------------------------------------------
     # Misc public methods
 

pandas/tests/test_frame.py

@@ -3307,6 +3307,22 @@ def test_combineMult(self):
         comb = self.empty.combineMult(self.frame)
         assert_frame_equal(comb, self.frame)
 
+    def test_combine_generic(self):
+        df1 = self.frame
+        df2 = self.frame.ix[:-5, ['A', 'B', 'C']]
+
+        combined = df1.combine(df2, np.add)
+        combined2 = df2.combine(df1, np.add)
+        self.assert_(combined['D'].isnull().all())
+        self.assert_(combined2['D'].isnull().all())
+
+        chunk = combined.ix[:-5, ['A', 'B', 'C']]
+        chunk2 = combined2.ix[:-5, ['A', 'B', 'C']]
+
+        exp = self.frame.ix[:-5, ['A', 'B', 'C']].reindex_like(chunk) * 2
+        assert_frame_equal(chunk, exp)
+        assert_frame_equal(chunk2, exp)
+
     def test_clip(self):
         median = self.frame.median().median()
 

pandas/tests/test_series.py

@@ -711,6 +711,13 @@ def test_append(self):
 
         self.assertRaises(Exception, self.ts.append, self.ts)
 
+    def test_all_any(self):
+        np.random.seed(12345)
+        ts = tm.makeTimeSeries()
+        bool_series = ts > 0
+        self.assert_(not bool_series.all())
+        self.assert_(bool_series.any())
+
     def test_operators(self):
         series = self.ts
         other = self.ts[::2]