FIX: Implement small Categorical fixups

This is a squashed commit which adresses several small issues
identified in the latest code review.

* As per dicussion in pandas-dev#8074,
  Categorical.unique() should return an array and not a Index object
  to make is consistant with other unique() implementations.

* Mark a few more categorical methods as internal

* Change reorder_levels and drop_unused_levels to inplace=False. As per
  the discussion in pandas-dev#8074, change the default of both methods
  to return a copy and only do the change inplace if told otherwise.

  Discusion in
  pandas-dev#8074 (comment)

* Make str(cat) more array like. As Categorical is more like
  np.ndarray and less similar to pd.Series, change the string
  representation of Categorical to be more like np.ndarray.

  The new str(cat) will only show at max 10 values.

  Also remove the "Categorical(" in front of an empty categorical
  -> either that's put in front of every str(cta) or none. It also missed
  the final ")".

* Test more nan handling in Categorical

* Add a testcase for groupby with two columns and unused categories

  During the devleopment of pandas new Categorical support, groupby
  with two columns (one worked) didn't include empty categories which
  omits rows with NaN. Add a testcase so this is checked.

  No codechanges nessesary, this bug was fixed at some other places
  before. See pandas-dev#8138 for more details.

Author: jankatins

doc/source/categorical.rst

@@ -298,12 +298,13 @@ This is even true for strings and numeric data:
 
 Reordering the levels is possible via the ``Categorical.reorder_levels(new_levels)``  or
 ``Series.cat.reorder_levels(new_levels)`` methods. All old levels must be included in the new
-levels.
+levels. Note that per default, this operation returns a new Series and you need to specify
+``inplace=True`` to do the change inplace!
 
 .. ipython:: python
 
-    s2 = pd.Series(pd.Categorical([1,2,3,1]))
-    s2.cat.reorder_levels([2,3,1])
+    s = pd.Series(pd.Categorical([1,2,3,1]))
+    s2 = s.cat.reorder_levels([2,3,1])
     s2
     s2.sort()
     s2
@@ -322,8 +323,8 @@ old levels:
 
 .. ipython:: python
 
-    s3 = pd.Series(pd.Categorical(["a","b","d"]))
-    s3.cat.reorder_levels(["a","b","c","d"])
+    s = pd.Series(pd.Categorical(["a","b","d"]))
+    s3 = s.cat.reorder_levels(["a","b","c","d"])
     s3
 
 
@@ -582,7 +583,7 @@ relevant columns back to `category` and assign the right levels and level orderi
     # rename the levels
     s.cat.levels = ["very good", "good", "bad"]
     # reorder the levels and add missing levels
-    s.cat.reorder_levels(["very bad", "bad", "medium", "good", "very good"])
+    s = s.cat.reorder_levels(["very bad", "bad", "medium", "good", "very good"])
     df = pd.DataFrame({"cats":s, "vals":[1,2,3,4,5,6]})
     csv = StringIO()
     df.to_csv(csv)
@@ -591,7 +592,7 @@ relevant columns back to `category` and assign the right levels and level orderi
     df2["cats"]
     # Redo the category
     df2["cats"] = df2["cats"].astype("category")
-    df2["cats"].cat.reorder_levels(["very bad", "bad", "medium", "good", "very good"])
+    df2["cats"].cat.reorder_levels(["very bad", "bad", "medium", "good", "very good"], inplace=True)
     df2.dtypes
     df2["cats"]
 
@@ -731,7 +732,7 @@ Series of type ``category`` this means that there is some danger to confuse both
     except Exception as e:
         print("Exception: " + str(e))
     # right
-    s.cat.reorder_levels([4,3,2,1])
+    s = s.cat.reorder_levels([4,3,2,1])
     print(s.cat.levels)
 
 See also the API documentation for :func:`pandas.Series.reorder_levels` and
@@ -742,8 +743,7 @@ Old style constructor usage
 
 I earlier versions, a `Categorical` could be constructed by passing in precomputed `codes`
 (called then `labels`) instead of values with levels. The `codes` are interpreted as pointers
-to the levels with `-1` as `NaN`. This usage is now deprecated and not available unless
-``compat=True`` is passed to the constructor of `Categorical`.
+to the levels with `-1` as `NaN`.
 
 .. ipython:: python
     :okwarning:

pandas/core/categorical.py

@@ -299,6 +299,8 @@ def from_array(cls, data):
         """
         Make a Categorical type from a single array-like object.
 
+        For internal compatibility with numpy arrays.
+
         Parameters
         ----------
         data : array-like
@@ -412,7 +414,7 @@ def _get_levels(self):
 
     levels = property(fget=_get_levels, fset=_set_levels, doc=_levels_doc)
 
-    def reorder_levels(self, new_levels, ordered=None):
+    def reorder_levels(self, new_levels, ordered=None, inplace=False):
         """ Reorders levels as specified in new_levels.
 
         `new_levels` must include all old levels but can also include new level items. In
@@ -432,27 +434,50 @@ def reorder_levels(self, new_levels, ordered=None):
         ordered : boolean, optional
            Whether or not the categorical is treated as a ordered categorical. If not given,
            do not change the ordered information.
+        inplace : bool (default: False)
+           Whether or not to reorder the levels inplace or return a copy of this categorical with
+           reordered levels.
+
+        Returns
+        -------
+        cat : Categorical with reordered levels.
         """
         new_levels = self._validate_levels(new_levels)
 
         if len(new_levels) < len(self._levels) or len(self._levels.difference(new_levels)):
             raise ValueError('Reordered levels must include all original levels')
-        values = self.__array__()
-        self._codes = _get_codes_for_values(values, new_levels)
-        self._levels = new_levels
+
+        cat = self if inplace else self.copy()
+        values = cat.__array__()
+        cat._codes = _get_codes_for_values(values, new_levels)
+        cat._levels = new_levels
         if not ordered is None:
-            self.ordered = ordered
+            cat.ordered = ordered
+        if not inplace:
+            return cat
 
-    def remove_unused_levels(self):
+    def remove_unused_levels(self, inplace=False):
         """ Removes levels which are not used.
 
-        The level removal is done inplace.
+        Parameters
+        ----------
+        inplace : bool (default: False)
+           Whether or not to drop unused levels inplace or return a copy of this categorical with
+           unused levels dropped.
+
+        Returns
+        -------
+        cat : Categorical with unused levels dropped.
+
         """
-        _used = sorted(np.unique(self._codes))
-        new_levels = self.levels.take(com._ensure_platform_int(_used))
+        cat = self if inplace else self.copy()
+        _used = sorted(np.unique(cat._codes))
+        new_levels = cat.levels.take(com._ensure_platform_int(_used))
         new_levels = _ensure_index(new_levels)
-        self._codes = _get_codes_for_values(self.__array__(), new_levels)
-        self._levels = new_levels
+        cat._codes = _get_codes_for_values(cat.__array__(), new_levels)
+        cat._levels = new_levels
+        if not inplace:
+            return cat
 
 
     __eq__ = _cat_compare_op('__eq__')
@@ -683,7 +708,14 @@ def view(self):
         return self
 
     def to_dense(self):
-        """ Return my 'dense' repr """
+        """Return my 'dense' representation
+
+        For internal compatibility with numpy arrays.
+
+        Returns
+        -------
+        dense : array
+        """
         return np.asarray(self)
 
     def fillna(self, fill_value=None, method=None, limit=None, **kwargs):
@@ -743,7 +775,10 @@ def fillna(self, fill_value=None, method=None, limit=None, **kwargs):
                            name=self.name, fastpath=True)
 
     def take_nd(self, indexer, allow_fill=True, fill_value=None):
-        """ Take the codes by the indexer, fill with the fill_value. """
+        """ Take the codes by the indexer, fill with the fill_value.
+
+        For internal compatibility with numpy arrays.
+        """
 
         # filling must always be None/nan here
         # but is passed thru internally
@@ -757,7 +792,10 @@ def take_nd(self, indexer, allow_fill=True, fill_value=None):
     take = take_nd
 
     def _slice(self, slicer):
-        """ Return a slice of myself. """
+        """ Return a slice of myself.
+
+        For internal compatibility with numpy arrays.
+        """
 
         # only allow 1 dimensional slicing, but can
         # in a 2-d case be passd (slice(None),....)
@@ -771,19 +809,21 @@ def _slice(self, slicer):
                            name=self.name, fastpath=True)
 
     def __len__(self):
+        """The length of this Categorical."""
         return len(self._codes)
 
     def __iter__(self):
+        """Returns an Iterator over the values of this Categorical."""
         return iter(np.array(self))
 
-    def _tidy_repr(self, max_vals=20):
+    def _tidy_repr(self, max_vals=10):
         num = max_vals // 2
         head = self[:num]._get_repr(length=False, name=False, footer=False)
         tail = self[-(max_vals - num):]._get_repr(length=False,
                                                   name=False,
                                                   footer=False)
 
-        result = '%s\n...\n%s' % (head, tail)
+        result = '%s, ..., %s' % (head[:-1], tail[1:])
         result = '%s\n%s' % (result, self._repr_footer())
 
         return compat.text_type(result)
@@ -840,17 +880,14 @@ def _get_repr(self, name=False, length=True, na_rep='NaN', footer=True):
 
     def __unicode__(self):
         """ Unicode representation. """
-        width, height = get_terminal_size()
-        max_rows = (height if get_option("display.max_rows") == 0
-                    else get_option("display.max_rows"))
-
-        if len(self._codes) > (max_rows or 1000):
-            result = self._tidy_repr(min(30, max_rows) - 4)
+        _maxlen = 10
+        if len(self._codes) > _maxlen:
+            result = self._tidy_repr(_maxlen)
         elif len(self._codes) > 0:
-            result = self._get_repr(length=len(self) > 50,
+            result = self._get_repr(length=len(self) > _maxlen,
                                     name=True)
         else:
-            result = 'Categorical([], %s' % self._get_repr(name=True,
+            result = '[], %s' % self._get_repr(name=True,
                                                            length=False,
                                                            footer=True,
                                                            ).replace("\n",", ")
@@ -1025,7 +1062,7 @@ def unique(self):
         -------
         unique values : array
         """
-        return self.levels
+        return np.asarray(self.levels)
 
     def equals(self, other):
         """
@@ -1113,8 +1150,11 @@ def _delegate_property_set(self, name, new_values):
         return setattr(self.categorical, name, new_values)
 
     def _delegate_method(self, name, *args, **kwargs):
+        from pandas import Series
         method = getattr(self.categorical, name)
-        return method(*args, **kwargs)
+        res = method(*args, **kwargs)
+        if not res is None:
+            return Series(res, index=self.index)
 
 CategoricalProperties._add_delegate_accessors(delegate=Categorical,
                                               accessors=["levels", "ordered"],

pandas/core/format.py

@@ -116,6 +116,9 @@ def to_string(self):
         fmt_values = self._get_formatted_values()
 
         result = ['%s' % i for i in fmt_values]
+        result = [i.strip() for i in result]
+        result = u(', ').join(result)
+        result = [u('[')+result+u(']')]
         if self.footer:
             footer = self._get_footer()
             if footer: