BUG: handle mixed-type frames carefully in unstack, GH pandas-dev#403

Author: wesm

RELEASE.rst

@@ -27,6 +27,11 @@ pandas 0.6.0
 
 **Release date:** Not yet released
 
+**API Changes**
+
+  - Arithmetic methods like `sum` will attempt to sum dtype=object values by
+    default instead of excluding them (GH #382)
+
 **New features / modules**
 
   - Add `melt` function to `pandas.core.reshape`
@@ -146,7 +151,9 @@ pandas 0.6.0
     depending on whether the passed function is a reduction (GH #389)
   - Always return NA/NaN from Series.min/max and DataFrame.min/max when all of a
     row/column/values are NA (GH #384)
-
+  - Enable partial setting with .ix / advanced indexing (GH #397)
+  - Handle mixed-type DataFrames correctly in unstack, do not lose type
+    information (GH #403)
 
 Thanks
 ------

pandas/core/frame.py

@@ -83,8 +83,9 @@
 
 _doc_exclude_na = "NA/null values are excluded"
 
-_numeric_only_doc = """numeric_only : boolean, default False
-    Include only float, int, boolean data
+_numeric_only_doc = """numeric_only : boolean, default None
+    Include only float, int, boolean data. If None, will attempt to use
+    everything, then use only numeric data
 """
 
 def _add_stat_doc(f, name, shortname, na_action=_doc_exclude_na,
@@ -2799,7 +2800,6 @@ def skew(self, axis=0, skipna=True, level=None):
     _add_stat_doc(skew, 'unbiased skewness', 'skew')
 
     def _reduce(self, op, axis=0, skipna=True, numeric_only=None):
-
         f = lambda x: op(x, axis=axis, skipna=skipna, copy=True)
         labels = self._get_agg_axis(axis)
         if numeric_only is None:

pandas/core/reshape.py

@@ -114,27 +114,32 @@ def _make_selectors(self):
     def get_result(self):
         # TODO: find a better way than this masking business
 
-        values, mask = self.get_new_values()
+        values, value_mask = self.get_new_values()
         columns = self.get_new_columns()
         index = self.get_new_index()
 
         # filter out missing levels
         if values.shape[1] > 0:
+            mask = value_mask.sum(0) > 0
             values = values[:, mask]
             columns = columns[mask]
 
         return DataFrame(values, index=index, columns=columns)
 
     def get_new_values(self):
+        return self._reshape_values(self.values)
+
+    def _reshape_values(self, values):
+        values = self.values
         # place the values
         length, width = self.full_shape
-        stride = self.values.shape[1]
+        stride = values.shape[1]
         result_width = width * stride
 
-        new_values = np.empty((length, result_width), dtype=self.values.dtype)
+        new_values = np.empty((length, result_width), dtype=values.dtype)
         new_mask = np.zeros((length, result_width), dtype=bool)
 
-        if issubclass(self.values.dtype.type, np.integer):
+        if issubclass(values.dtype.type, np.integer):
             new_values = new_values.astype(float)
 
         new_values.fill(np.nan)
@@ -148,7 +153,7 @@ def get_new_values(self):
             mask_chunk.flat[self.mask] = True
 
         new_values = new_values.take(self.unique_groups, axis=0)
-        return new_values, new_mask.sum(0) > 0
+        return new_values, new_mask
 
     def get_new_columns(self):
         if self.value_columns is None:
@@ -284,12 +289,43 @@ def _slow_pivot(index, columns, values):
 
 def unstack(obj, level):
     if isinstance(obj, DataFrame):
-        columns = obj.columns
+        return _unstack_frame(obj, level)
+    else:
+        unstacker = _Unstacker(obj.values, obj.index, level=level)
+        return unstacker.get_result()
+
+def _unstack_frame(obj, level):
+    from pandas.core.internals import BlockManager, make_block
+
+    if obj._is_mixed_type:
+        unstacker = _Unstacker(np.empty(obj.shape, dtype=bool), # dummy
+                               obj.index, level=level,
+                               value_columns=obj.columns)
+        new_columns = unstacker.get_new_columns()
+        new_index = unstacker.get_new_index()
+        new_axes = [new_columns, new_index]
+
+        new_blocks = []
+        mask_blocks = []
+        for blk in obj._data.blocks:
+            bunstacker = _Unstacker(blk.values.T, obj.index, level=level,
+                                    value_columns=blk.items)
+            new_items = bunstacker.get_new_columns()
+            new_values, mask = bunstacker.get_new_values()
+
+            mblk = make_block(mask.T, new_items, new_columns)
+            mask_blocks.append(mblk)
+
+            newb = make_block(new_values.T, new_items, new_columns)
+            new_blocks.append(newb)
+
+        result = DataFrame(BlockManager(new_blocks, new_axes))
+        mask_frame = DataFrame(BlockManager(mask_blocks, new_axes))
+        return result.ix[:, mask_frame.sum(0) > 0]
     else:
-        columns = None
-    unstacker = _Unstacker(obj.values, obj.index, level=level,
-                           value_columns=columns)
-    return unstacker.get_result()
+        unstacker = _Unstacker(obj.values, obj.index, level=level,
+                               value_columns=obj.columns)
+        return unstacker.get_result()
 
 def stack(frame, level=-1, dropna=True):
     """

pandas/tests/test_multilevel.py

@@ -679,6 +679,16 @@ def test_partial_set(self):
         exp.ix[2000].values[:] = 5
         assert_frame_equal(df, exp)
 
+    def test_unstack_preserve_types(self):
+        # GH #403
+        self.ymd['E'] = 'foo'
+        self.ymd['F'] = 2
+
+        unstacked = self.ymd.unstack('month')
+        self.assert_(unstacked['A', 1].dtype == np.float64)
+        self.assert_(unstacked['E', 1].dtype == np.object_)
+        self.assert_(unstacked['F', 1].dtype == np.float64)
+
 if __name__ == '__main__':
 
     # unittest.main()