BUG: test coverage, groupby bug fixes

Author: wesm

pandas/core/frame.py

@@ -240,7 +240,7 @@ def na_op(x, y):
                 mask = notnull(xrav)
                 result[mask] = op(np.array(list(xrav[mask])), y)
 
-            if op == operator.ne:
+            if op == operator.ne:  # pragma: no cover
                 np.putmask(result, -mask, True)
             else:
                 np.putmask(result, -mask, False)
@@ -1869,12 +1869,12 @@ def xs(self, key, axis=0, level=None, copy=True):
         if np.isscalar(loc):
             new_values = self._data.fast_2d_xs(loc, copy=copy)
             return Series(new_values, index=self.columns, name=key)
-        elif isinstance(loc, slice) or loc.dtype == np.bool_:
+        else: # isinstance(loc, slice) or loc.dtype == np.bool_:
             result = self[loc]
             result.index = new_index
             return result
-        else:
-            return self.take(loc)
+        # else:
+        #     return self.take(loc)
 
     def lookup(self, row_labels, col_labels):
         """

pandas/core/groupby.py

@@ -345,12 +345,11 @@ def ohlc(self):
 
     def nth(self, n):
         def picker(arr):
-            if arr is not None:
-                n_ok_pos = n >= 0 and len(arr) > n
-                n_ok_neg = n < 0 and len(arr) >= n
-                if n_ok_pos or n_ok_neg:
-                    return arr.iget(n)
-            return np.nan
+            arr = arr[com.notnull(arr)]
+            if len(arr) >= n + 1:
+                return arr.iget(n)
+            else:
+                return np.nan
         return self.agg(picker)
 
     def _cython_agg_general(self, how):
@@ -656,11 +655,13 @@ def aggregate(self, values, how, axis=0):
         arity = self._cython_arity.get(how, 1)
 
         vdim = values.ndim
+        swapped = False
         if vdim == 1:
             values = values[:, None]
             out_shape = (self.ngroups, arity)
         else:
             if axis > 0:
+                swapped = True
                 values = values.swapaxes(0, axis)
             if arity > 1:
                 raise NotImplementedError
@@ -673,8 +674,11 @@ def aggregate(self, values, how, axis=0):
         result = self._aggregate(result, counts, values, how)
 
         if self._filter_empty_groups:
-            result = lib.row_bool_subset(result,
-                                         (counts > 0).view(np.uint8))
+            if result.ndim == 2:
+                result = lib.row_bool_subset(result,
+                                             (counts > 0).view(np.uint8))
+            else:
+                result = result[counts > 0]
 
         if vdim == 1 and arity == 1:
             result = result[:, 0]
@@ -685,7 +689,7 @@ def aggregate(self, values, how, axis=0):
         else:
             names = None
 
-        if axis > 0:
+        if swapped:
             result = result.swapaxes(0, axis)
 
         return result, names
@@ -700,7 +704,8 @@ def _aggregate(self, result, counts, values, how):
             raise NotImplementedError
         elif values.ndim > 2:
             for i, chunk in enumerate(values.transpose(2, 0, 1)):
-                agg_func(result[:, :, i], counts, chunk, comp_ids)
+                agg_func(result[:, :, i], counts, chunk.squeeze(),
+                         comp_ids)
         else:
             agg_func(result, counts, values, comp_ids)
 
@@ -942,10 +947,6 @@ def __init__(self, index, grouper=None, name=None, level=None,
         # pre-computed
         self._was_factor = False
 
-        # did we pass a custom grouper object? Do nothing
-        if isinstance(grouper, Grouper):
-            return
-
         if level is not None:
             if not isinstance(level, int):
                 assert(level in index.names)
@@ -1349,7 +1350,13 @@ def _wrap_agged_blocks(self, blocks):
         obj = self._obj_with_exclusions
 
         new_axes = list(obj._data.axes)
-        new_axes[self.axis] = self.grouper.result_index
+
+        # more kludge
+        if self.axis == 0:
+            new_axes[0], new_axes[1] = new_axes[1], self.grouper.result_index
+        else:
+            new_axes[self.axis] = self.grouper.result_index
+
         mgr = BlockManager(blocks, new_axes)
 
         new_obj = type(obj)(mgr)
@@ -1693,7 +1700,7 @@ def _transform_item_by_item(self, obj, wrapper):
             except Exception:
                 pass
 
-        if len(output) == 0:
+        if len(output) == 0:  # pragma: no cover
             raise TypeError('Transform function invalid for data types')
 
         columns = obj.columns
@@ -1769,12 +1776,6 @@ def _wrap_aggregated_output(self, output, names=None):
 
         return result
 
-    def _post_process_cython_aggregate(self, obj):
-        # undoing kludge from below
-        if self.axis == 0:
-            obj = obj.T
-        return obj
-
     def _wrap_agged_blocks(self, blocks):
         obj = self._obj_with_exclusions
 
@@ -1827,8 +1828,6 @@ def _iterate_slices(self):
             slicer = lambda x: self.obj[x]
         else:
             raise NotImplementedError
-            # slice_axis = self.obj.index
-            # slicer = lambda x: self.obj.xs(x, axis=self.axis)
 
         for val in slice_axis:
             if val in self.exclusions:
@@ -1857,7 +1856,6 @@ def aggregate(self, arg, *args, **kwargs):
         return self._aggregate_generic(arg, *args, **kwargs)
 
     def _wrap_generic_output(self, result, obj):
-
         new_axes = list(obj.axes)
         new_axes[self.axis] = self.grouper.result_index
 
@@ -1882,8 +1880,6 @@ def _aggregate_item_by_item(self, func, *args, **kwargs):
                     result[item] = itemg.aggregate(func, *args, **kwargs)
                 except (ValueError, TypeError):
                     raise
-                    # cannot_agg.append(item)
-                    # continue
             new_axes = list(obj.axes)
             new_axes[self.axis] = self.grouper.result_index
             return Panel._from_axes(result, new_axes)
@@ -1892,16 +1888,6 @@ def _aggregate_item_by_item(self, func, *args, **kwargs):
 
     def _wrap_aggregated_output(self, output, names=None):
         raise NotImplementedError
-        new_axes = list(self._obj_with_exclusions.axes)
-        new_axes[self.axis] = self.grouper.result_index
-
-        result = Panel(output, index=self.grouper.result_index,
-                       columns=output_keys)
-
-        if self.axis > 0:
-            result = result.swapaxes(0, self.axis)
-
-        return result
 
 
 class NDArrayGroupBy(GroupBy):

pandas/core/series.py

@@ -2736,7 +2736,7 @@ def _resolve_offset(freq, kwds):
         offset = freq
         warn = False
 
-    if warn and _SHOW_WARNINGS:
+    if warn and _SHOW_WARNINGS:  # pragma: no cover
         import warnings
         warnings.warn("'timeRule' and 'offset' parameters are deprecated,"
                       " please use 'freq' instead",

pandas/stats/misc.py

@@ -4,9 +4,7 @@
 from pandas.core.api import Series, DataFrame, isnull, notnull
 from pandas.core.series import remove_na
 
-from pandas.tools.tile import (bucket, bucketcat, bucketpanel,
-                               quantileTS)
-
+from pandas.tools.tile import quantileTS
 
 def zscore(series):
     return (series - series.mean()) / np.std(series, ddof = 0)
@@ -96,3 +94,196 @@ def percentileRank(frame, column=None, kind='mean'):
                 results.setdefault(date, {})[column] = fun(xs, xs[column])
         results = DataFrame(results).T
     return results
+
+
+def bucket(series, k, by=None):
+    """
+    Produce DataFrame representing quantiles of a Series
+
+    Parameters
+    ----------
+    series : Series
+    k : int
+        number of quantiles
+    by : Series or same-length array
+        bucket by value
+
+    Returns
+    -------
+    DataFrame
+    """
+    if by is None:
+        by = series
+    else:
+        by = by.reindex(series.index)
+
+    split = _split_quantile(by, k)
+    mat = np.empty((len(series), k), dtype=float) * np.NaN
+
+    for i, v in enumerate(split):
+        mat[:, i][v] = series.take(v)
+
+    return DataFrame(mat, index=series.index, columns=np.arange(k) + 1)
+
+def _split_quantile(arr, k):
+    arr = np.asarray(arr)
+    mask = np.isfinite(arr)
+    order = arr[mask].argsort()
+    n = len(arr)
+
+    return np.array_split(np.arange(n)[mask].take(order), k)
+
+def bucketcat(series, cats):
+    """
+    Produce DataFrame representing quantiles of a Series
+
+    Parameters
+    ----------
+    series : Series
+    cat : Series or same-length array
+        bucket by category; mutually exxlusive with 'by'
+
+    Returns
+    -------
+    DataFrame
+    """
+    if not isinstance(series, Series):
+        series = Series(series, index=np.arange(len(series)))
+
+    cats = np.asarray(cats)
+
+    unique_labels = np.unique(cats)
+    unique_labels = unique_labels[com.notnull(unique_labels)]
+
+    # group by
+    data = {}
+
+    for label in unique_labels:
+        data[label] = series[cats == label]
+
+    return DataFrame(data, columns=unique_labels)
+
+def bucketpanel(series, bins=None, by=None, cat=None):
+    """
+    Bucket data by two Series to create summary panel
+
+    Parameters
+    ----------
+    series : Series
+    bins : tuple (length-2)
+        e.g. (2, 2)
+    by : tuple of Series
+        bucket by value
+    cat : tuple of Series
+        bucket by category; mutually exxlusive with 'by'
+
+    Returns
+    -------
+    DataFrame
+    """
+    use_by = by is not None
+    use_cat = cat is not None
+
+    if use_by and use_cat:
+        raise Exception('must specify by or cat, but not both')
+    elif use_by:
+        if len(by) != 2:
+            raise Exception('must provide two bucketing series')
+
+        xby, yby = by
+        xbins, ybins = bins
+
+        return _bucketpanel_by(series, xby, yby, xbins, ybins)
+
+    elif use_cat:
+        xcat, ycat = cat
+        return _bucketpanel_cat(series, xcat, ycat)
+    else:
+        raise Exception('must specify either values or categories to bucket by')
+
+def _bucketpanel_by(series, xby, yby, xbins, ybins):
+    xby = xby.reindex(series.index)
+    yby = yby.reindex(series.index)
+
+    xlabels = _bucket_labels(xby.reindex(series.index), xbins)
+    ylabels = _bucket_labels(yby.reindex(series.index), ybins)
+
+    labels = _uniquify(xlabels, ylabels, xbins, ybins)
+
+    mask = com.isnull(labels)
+    labels[mask] = -1
+
+    unique_labels = np.unique(labels)
+    bucketed = bucketcat(series, labels)
+
+    _ulist = list(labels)
+    index_map = dict((x, _ulist.index(x)) for x in unique_labels)
+
+    def relabel(key):
+        pos = index_map[key]
+
+        xlab = xlabels[pos]
+        ylab = ylabels[pos]
+
+        return '%sx%s' % (int(xlab) if com.notnull(xlab) else 'NULL',
+                          int(ylab) if com.notnull(ylab) else 'NULL')
+
+    return bucketed.rename(columns=relabel)
+
+def _bucketpanel_cat(series, xcat, ycat):
+    xlabels, xmapping = _intern(xcat)
+    ylabels, ymapping = _intern(ycat)
+
+    shift = 10 ** (np.ceil(np.log10(ylabels.max())))
+    labels = xlabels * shift + ylabels
+
+    sorter = labels.argsort()
+    sorted_labels = labels.take(sorter)
+    sorted_xlabels = xlabels.take(sorter)
+    sorted_ylabels = ylabels.take(sorter)
+
+    unique_labels = np.unique(labels)
+    unique_labels = unique_labels[com.notnull(unique_labels)]
+
+    locs = sorted_labels.searchsorted(unique_labels)
+    xkeys = sorted_xlabels.take(locs)
+    ykeys = sorted_ylabels.take(locs)
+
+    stringified = ['(%s, %s)' % arg
+                   for arg in zip(xmapping.take(xkeys), ymapping.take(ykeys))]
+
+    result = bucketcat(series, labels)
+    result.columns = stringified
+
+    return result
+
+def _intern(values):
+    # assumed no NaN values
+    values = np.asarray(values)
+
+    uniqued = np.unique(values)
+    labels = uniqued.searchsorted(values)
+    return labels, uniqued
+
+
+def _uniquify(xlabels, ylabels, xbins, ybins):
+    # encode the stuff, create unique label
+    shifter = 10 ** max(xbins, ybins)
+    _xpiece = xlabels * shifter
+    _ypiece = ylabels
+
+    return _xpiece + _ypiece
+
+def _bucket_labels(series, k):
+    arr = np.asarray(series)
+    mask = np.isfinite(arr)
+    order = arr[mask].argsort()
+    n = len(series)
+
+    split = np.array_split(np.arange(n)[mask].take(order), k)
+
+    mat = np.empty(n, dtype=float) * np.NaN
+    for i, v in enumerate(split):
+        mat[v] = i
+
+    return mat + 1