BUG: GroupBy.apply bug with differently indexed MultiIndex objects, test coverage

Author: wesm

RELEASE.rst

@@ -194,6 +194,8 @@ feedback on the library.
   - Can pass level name to `DataFrame.stack`
   - Support set operations between MultiIndex and Index
   - Fix many corner cases in MultiIndex set operations
+  - Fix MultiIndex-handling bug with GroupBy.apply when returned groups are not
+    indexed the same
 
 Thanks
 ------

pandas/core/common.py

@@ -416,7 +416,7 @@ def __call__(self, num):
 
         sign = 1
 
-        if dnum < 0:
+        if dnum < 0:  # pragma: no cover
             sign = -1
             dnum = -dnum
 
@@ -439,7 +439,7 @@ def __call__(self, num):
 
         mant = sign*dnum/(10**pow10)
 
-        if self.precision is None:
+        if self.precision is None:  # pragma: no cover
             format_str = u"%g%s"
         elif self.precision == 0:
             format_str = u"%i%s"

pandas/core/groupby.py

@@ -412,8 +412,13 @@ def _python_apply_general(self, func, *args, **kwargs):
         not_indexed_same = False
         for key, group in self:
             group.name = key
+
+            # group might be modified
+            group_axes = _get_axes(group)
+
             res = func(group, *args, **kwargs)
-            if not _is_indexed_like(res, group):
+
+            if not _is_indexed_like(res, group_axes):
                 not_indexed_same = True
 
             result_keys.append(key)
@@ -460,18 +465,19 @@ def groupby(obj, by, **kwds):
     return klass(obj, by, **kwds)
 groupby.__doc__ = GroupBy.__doc__
 
-def _is_indexed_like(obj, other):
+def _get_axes(group):
+    if isinstance(group, Series):
+        return [group.index]
+    else:
+        return group.axes
+
+def _is_indexed_like(obj, axes):
     if isinstance(obj, Series):
-        if not isinstance(other, Series):
+        if len(axes) > 1:
             return False
-        return obj.index.equals(other.index)
+        return obj.index.equals(axes[0])
     elif isinstance(obj, DataFrame):
-        if isinstance(other, Series):
-            return obj.index.equals(other.index)
-
-        # deal with this when a case arises
-        assert(isinstance(other, DataFrame))
-        return obj._indexed_same(other)
+        return obj.index.equals(axes[0])
 
     return False
 
@@ -1093,11 +1099,7 @@ def _concat_frames(frames, index, columns=None, axis=0):
         return result.reindex(index=index, columns=columns)
 
 def _concat_indexes(indexes):
-    if len(indexes) == 1:
-        new_index = indexes[0]
-    else:
-        new_index = indexes[0].append(indexes[1:])
-    return new_index
+    return indexes[0].append(indexes[1:])
 
 def _concat_frames_hierarchical(frames, keys, groupings, axis=0):
     if axis == 0:
@@ -1135,8 +1137,14 @@ def _make_concat_multiindex(indexes, keys, groupings):
                 to_concat.append(np.repeat(k, len(index)))
             label_list.append(np.concatenate(to_concat))
 
-        # these go in the last level
-        label_list.append(np.concatenate(indexes))
+        concat_index = _concat_indexes(indexes)
+
+        # these go at the end
+        if isinstance(concat_index, MultiIndex):
+            for level in range(concat_index.nlevels):
+                label_list.append(concat_index.get_level_values(level))
+        else:
+            label_list.append(concat_index.values)
 
         return MultiIndex.from_arrays(label_list)
 

pandas/tests/test_frame.py

@@ -11,6 +11,7 @@
 from numpy.random import randn
 import numpy as np
 
+import pandas.core.common as common
 import pandas.core.datetools as datetools
 from pandas.core.index import NULL_INDEX
 from pandas.core.api import (DataFrame, Index, Series, notnull, isnull,
@@ -1245,10 +1246,26 @@ def test_repr(self):
                                index=np.arange(50))
         foo = repr(unsortable)
 
-        import pandas.core.common as common
         common.set_printoptions(precision=3, column_space=10)
         repr(self.frame)
 
+    def test_eng_float_formatter(self):
+        self.frame.ix[5] = 0
+
+        common.set_eng_float_format()
+
+        repr(self.frame)
+
+        common.set_eng_float_format(use_eng_prefix=True)
+
+        repr(self.frame)
+
+        common.set_eng_float_format(precision=0)
+
+        repr(self.frame)
+
+        common.set_printoptions(precision=4)
+
     def test_repr_tuples(self):
         buf = StringIO()
 

pandas/tests/test_groupby.py

@@ -913,6 +913,31 @@ def f(group):
 
         assert_frame_equal(result, expected)
 
+    def test_apply_corner(self):
+        result = self.tsframe.groupby(lambda x: x.year).apply(lambda x: x * 2)
+        expected = self.tsframe * 2
+        assert_frame_equal(result, expected)
+
+    def test_transform_mixed_type(self):
+        index = MultiIndex.from_arrays([[0, 0, 0, 1, 1, 1],
+                                        [1, 2, 3, 1, 2, 3]])
+        df = DataFrame({'d' : [1.,1.,1.,2.,2.,2.],
+                        'c' : np.tile(['a','b','c'], 2),
+                        'v' : np.arange(1., 7.)}, index=index)
+
+        def f(group):
+            group['g'] = group['d'] * 2
+            return group[:1]
+
+        grouped = df.groupby('c')
+        result = grouped.apply(f)
+
+        self.assert_(result['d'].dtype == np.float64)
+
+        for key, group in grouped:
+            res = f(group)
+            assert_frame_equal(res, result.ix[key])
+
 class TestPanelGroupBy(unittest.TestCase):
 
     def setUp(self):

pandas/tests/test_index.py

@@ -944,16 +944,19 @@ def test_diff(self):
         result = self.index - self.index
         expected = self.index[:0]
         self.assert_(result.equals(expected))
+        self.assertEqual(result.names, self.index.names)
 
         # empty difference: superset
         result = self.index[-3:] - self.index
         expected = self.index[:0]
         self.assert_(result.equals(expected))
+        self.assertEqual(result.names, self.index.names)
 
         # empty difference: degenerate
         result = self.index[:0] - self.index
         expected = self.index[:0]
         self.assert_(result.equals(expected))
+        self.assertEqual(result.names, self.index.names)
 
         # names not the same
         chunklet = self.index[-3:]