BUG: fix groupby bug/segfault with NAs in hierarchical level. fix MultiIndex tuple with NA issue. close #2616

Author: wesm

RELEASE.rst

@@ -65,6 +65,9 @@ pandas 0.10.1
   - More robust detection of being in IPython session for wide DataFrame
     console formatting (GH2585_)
   - Fix platform issues with ``file:///`` in unit test (#2564)
+  - Fix bug and possible segfault when grouping by hierarchical level that
+    contains NA values (GH2616_)
+  - Ensure that MultiIndex tuples can be constructed with NAs (seen in #2616)
 
 **API Changes**
 
@@ -78,6 +81,7 @@ pandas 0.10.1
 .. _GH2576: https://github.com/pydata/pandas/issues/2576
 .. _GH2585: https://github.com/pydata/pandas/issues/2585
 .. _GH2604: https://github.com/pydata/pandas/issues/2604
+.. _GH2616: https://github.com/pydata/pandas/issues/2616
 
 pandas 0.10.0
 =============

pandas/core/groupby.py

@@ -1109,6 +1109,15 @@ def __init__(self, index, grouper=None, name=None, level=None,
                 # all levels may not be observed
                 labels, uniques = algos.factorize(inds, sort=True)
 
+                if len(uniques) > 0 and uniques[0] == -1:
+                    # handle NAs
+                    mask = inds != -1
+                    ok_labels, uniques = algos.factorize(inds[mask], sort=True)
+
+                    labels = np.empty(len(inds), dtype=inds.dtype)
+                    labels[mask] = ok_labels
+                    labels[-mask] = -1
+
                 if len(uniques) < len(level_index):
                     level_index = level_index.take(uniques)
 

pandas/core/index.py

@@ -1467,7 +1467,7 @@ def values(self):
 
             values = []
             for lev, lab in zip(self.levels, self.labels):
-                taken = ndtake(lev.values, lab)
+                taken = com.take_1d(lev.values, lab)
                 # Need to box timestamps, etc.
                 if hasattr(lev, '_box_values'):
                     taken = lev._box_values(taken)

pandas/tests/test_groupby.py

@@ -1157,6 +1157,27 @@ def test_groupby_level(self):
         # raise exception for non-MultiIndex
         self.assertRaises(ValueError, self.df.groupby, level=1)
 
+    def test_groupby_level_with_nas(self):
+        index = MultiIndex(levels=[[1, 0], [0, 1, 2, 3]],
+                           labels=[[1, 1, 1, 1, 0, 0, 0, 0],
+                                   [0, 1, 2, 3, 0, 1, 2, 3]])
+
+        # factorizing doesn't confuse things
+        s = Series(np.arange(8.), index=index)
+        result = s.groupby(level=0).sum()
+        expected = Series([22., 6.], index=[1, 0])
+        assert_series_equal(result, expected)
+
+        index = MultiIndex(levels=[[1, 0], [0, 1, 2, 3]],
+                           labels=[[1, 1, 1, 1, -1, 0, 0, 0],
+                                   [0, 1, 2, 3, 0, 1, 2, 3]])
+
+        # factorizing doesn't confuse things
+        s = Series(np.arange(8.), index=index)
+        result = s.groupby(level=0).sum()
+        expected = Series([18., 6.], index=[1, 0])
+        assert_series_equal(result, expected)
+
     def test_groupby_level_apply(self):
         frame = self.mframe
 

pandas/tests/test_multilevel.py

@@ -1684,6 +1684,14 @@ def test_assign_index_sequences(self):
         df.index = l
         repr(df)
 
+    def test_tuples_have_na(self):
+        index = MultiIndex(levels=[[1, 0], [0, 1, 2, 3]],
+                           labels=[[1, 1, 1, 1, -1, 0, 0, 0],
+                                   [0, 1, 2, 3, 0, 1, 2, 3]])
+
+        self.assertTrue(isnull(index[4][0]))
+        self.assertTrue(isnull(index.values[4][0]))
+
 
 if __name__ == '__main__':