Categorical: fix describe with np.nan

Author: jankatins

pandas/core/categorical.py

@@ -931,12 +931,25 @@ def describe(self):
             'values' : self._codes }
                            ).groupby('codes').count()
 
-        counts.index = self.levels.take(counts.index)
-        counts = counts.reindex(self.levels)
         freqs = counts / float(counts.sum())
 
         from pandas.tools.merge import concat
         result = concat([counts,freqs],axis=1)
-        result.index.name = 'levels'
         result.columns = ['counts','freqs']
+
+        # Up to now we have codes -> fill in the levels
+        # object in case we need to handle NaNs
+        levels = np.asarray(self.levels, dtype=object)
+        # use arange to also include not used levels
+        index = np.arange(0, len(levels))
+        # handle nan
+        if -1 in result.index:
+            # take[...,-1] returns the last element. So put np.nan there...
+            levels = np.append(levels, np.nan)
+            # also sort the -1 to the last position in the index
+            index = np.append(index, -1)
+        result = result.reindex(index)
+        result.index = levels.take(result.index)
+        result.index.name = 'levels'
+
         return result

pandas/tests/test_categorical.py

@@ -217,6 +217,16 @@ def test_describe(self):
                                             ).set_index('levels')
         tm.assert_frame_equal(desc, expected)
 
+        # check unused levels
+        cat = self.factor.copy()
+        cat.levels = ["a","b","c","d"]
+        desc = cat.describe()
+        expected = DataFrame.from_dict(dict(counts=[3, 2, 3, np.nan],
+                                            freqs=[3/8., 2/8., 3/8., np.nan],
+                                            levels=['a', 'b', 'c', 'd'])
+                                            ).set_index('levels')
+        tm.assert_frame_equal(desc, expected)
+
         # check an integer one
         desc = Categorical([1,2,3,1,2,3,3,2,1,1,1]).describe()
         expected = DataFrame.from_dict(dict(counts=[5, 3, 3],
@@ -226,6 +236,29 @@ def test_describe(self):
                                             ).set_index('levels')
         tm.assert_frame_equal(desc, expected)
 
+        # https://github.com/pydata/pandas/issues/3678
+        # describe should work with NaN
+        cat = pd.Categorical([np.nan,1, 2, 2])
+        desc = cat.describe()
+        expected = DataFrame.from_dict(dict(counts=[1, 2, 1],
+                                            freqs=[1/4., 2/4., 1/4.],
+                                            levels=[1,2,np.nan]
+                                            )
+                                            ).set_index('levels')
+        tm.assert_frame_equal(desc, expected)
+
+        # having NaN as level and as "not available" should also print two NaNs in describe!
+        cat = pd.Categorical([np.nan,1, 2, 2])
+        cat.levels = [1,2,np.nan]
+        desc = cat.describe()
+        expected = DataFrame.from_dict(dict(counts=[1, 2, np.nan, 1],
+                                            freqs=[1/4., 2/4., np.nan, 1/4.],
+                                            levels=[1,2,np.nan,np.nan]
+                                            )
+                                            ).set_index('levels')
+        tm.assert_frame_equal(desc, expected)
+
+
     def test_print(self):
         expected = [" a", " b", " b", " a", " a", " c", " c", " c",
                     "Levels (3, object): [a < b < c]"]