BUG: categorical fixups, related GH3678

Author: jreback

pandas/core/categorical.py

@@ -220,13 +220,12 @@ def __init__(self, values, levels=None, ordered=None, name=None, fastpath=False,
             inferred = com._possibly_infer_to_datetimelike(values)
             if not isinstance(inferred, np.ndarray):
                 from pandas.core.series import _sanitize_array
-                safe_dtype = None
-                if isinstance(values, list) and np.nan in values:
-                    # On list with NaNs, int values will be converted to float. Use "object" dtype
-                    # to prvent this. In the end objects will be casted to int/... in the level
-                    # assignment step.
-                    safe_dtype = "object"
-                values = _sanitize_array(values, None, dtype=safe_dtype)
+
+                # On list with NaNs, int values will be converted to float. Use "object" dtype
+                # to prvent this. In the end objects will be casted to int/... in the level
+                # assignment step.
+                dtype = object if com.isnull(values).any() else None
+                values = _sanitize_array(values, None, dtype=dtype)
 
         if levels is None:
             # object is needed to preserve ints in case we have np.nan in values
@@ -932,24 +931,20 @@ def describe(self):
                            ).groupby('codes').count()
 
         freqs = counts / float(counts.sum())
-
         from pandas.tools.merge import concat
         result = concat([counts,freqs],axis=1)
         result.columns = ['counts','freqs']
 
-        # Up to now we have codes -> fill in the levels
-        # object in case we need to handle NaNs
-        levels = np.asarray(self.levels, dtype=object)
-        # use arange to also include not used levels
-        index = np.arange(0, len(levels))
-        # handle nan
-        if -1 in result.index:
-            # take[...,-1] returns the last element. So put np.nan there...
-            levels = np.append(levels, np.nan)
-            # also sort the -1 to the last position in the index
-            index = np.append(index, -1)
-        result = result.reindex(index)
-        result.index = levels.take(result.index)
-        result.index.name = 'levels'
+        check = counts.index == -1
+        if check.any():
+            l = len(self.levels) if com.isnull(self.levels).any() else len(self.levels)+1
+            index = np.arange(0,l,dtype=object)
+            index[~check] = self.levels.take(counts.index[~check])
+            index[check] = np.nan
+            result.index = index
+        else:
+            result.index = self.levels.take(counts.index)
+            result = result.reindex(self.levels)
 
+        result.index.name = 'levels'
         return result

pandas/tests/test_categorical.py

@@ -240,20 +240,20 @@ def test_describe(self):
         # describe should work with NaN
         cat = pd.Categorical([np.nan,1, 2, 2])
         desc = cat.describe()
-        expected = DataFrame.from_dict(dict(counts=[1, 2, 1],
-                                            freqs=[1/4., 2/4., 1/4.],
-                                            levels=[1,2,np.nan]
+        expected = DataFrame.from_dict(dict(counts=[1, 1, 2],
+                                            freqs=[1/4., 1/4., 2/4.],
+                                            levels=[np.nan,1,2]
                                             )
                                             ).set_index('levels')
         tm.assert_frame_equal(desc, expected)
 
-        # having NaN as level and as "not available" should also print two NaNs in describe!
+        # FIXME: this seemse very wrong: having NaN as level and as "not available" should also print two NaNs in describe!
         cat = pd.Categorical([np.nan,1, 2, 2])
         cat.levels = [1,2,np.nan]
         desc = cat.describe()
-        expected = DataFrame.from_dict(dict(counts=[1, 2, np.nan, 1],
-                                            freqs=[1/4., 2/4., np.nan, 1/4.],
-                                            levels=[1,2,np.nan,np.nan]
+        expected = DataFrame.from_dict(dict(counts=[1, 1, 2],
+                                            freqs=[1/4., 1/4., 2/4.],
+                                            levels=[np.nan,1,2]
                                             )
                                             ).set_index('levels')
         tm.assert_frame_equal(desc, expected)