API/ENH: IntervalIndex

closes pandas-dev#7640
closes pandas-dev#8625

Author: shoyer

pandas/_libs/hashtable.pyx

@@ -39,9 +39,6 @@ PyDateTime_IMPORT
 cdef extern from "Python.h":
     int PySlice_Check(object)
 
-cdef size_t _INIT_VEC_CAP = 128
-
-
 include "hashtable_class_helper.pxi"
 include "hashtable_func_helper.pxi"
 

pandas/_libs/lib.pyx

@@ -245,6 +245,7 @@ cpdef bint isscalar(object val):
     - instances of datetime.timedelta
     - Period
     - instances of decimal.Decimal
+    - Interval
 
     """
 
@@ -258,7 +259,8 @@ cpdef bint isscalar(object val):
             or PyDelta_Check(val)
             or PyTime_Check(val)
             or util.is_period_object(val)
-            or is_decimal(val))
+            or is_decimal(val),
+            or is_interval(val))
 
 
 def item_from_zerodim(object val):
@@ -1896,4 +1898,6 @@ cdef class BlockPlacement:
 
 include "reduce.pyx"
 include "properties.pyx"
+include "interval.pyx"
+include "intervaltree.pyx"
 include "inference.pyx"

pandas/_libs/src/inference.pyx

@@ -429,6 +429,10 @@ def infer_dtype(object value):
         if is_period_array(values):
             return 'period'
 
+    elif is_interval(val):
+        if is_interval_array_fixed_closed(values):
+            return 'interval'
+
     for i in range(n):
         val = util.get_value_1d(values, i)
         if (util.is_integer_object(val) and
@@ -879,6 +883,23 @@ cpdef bint is_period_array(ndarray[object] values):
             return False
     return null_count != n
 
+cdef inline bint is_interval(object o):
+    return isinstance(o, Interval)
+
+def is_interval_array_fixed_closed(ndarray[object] values):
+    cdef Py_ssize_t i, n = len(values)
+    cdef str closed
+    if n == 0:
+        return False
+    for i in range(n):
+        if not is_interval(values[i]):
+            return False
+        if i == 0:
+            closed = values[0].closed
+        elif closed != values[i].closed:
+            return False
+    return True
+
 
 cdef extern from "parse_helper.h":
     inline int floatify(object, double *result, int *maybe_int) except -1

pandas/core/algorithms.py

@@ -528,7 +528,6 @@ def value_counts(values, sort=True, ascending=False, normalize=False,
             cat, bins = cut(values, bins, retbins=True)
         except TypeError:
             raise TypeError("bins argument only works with numeric data.")
-        values = cat.codes
 
     if is_categorical_dtype(values) or is_sparse(values):
 

pandas/core/api.py

@@ -12,6 +12,7 @@
 from pandas.core.index import (Index, CategoricalIndex, Int64Index,
                                UInt64Index, RangeIndex, Float64Index,
                                MultiIndex)
+from pandas.core.interval import Interval, IntervalIndex
 
 from pandas.core.series import Series
 from pandas.core.frame import DataFrame

pandas/core/groupby.py

@@ -43,8 +43,7 @@
 from pandas.core.categorical import Categorical
 from pandas.core.frame import DataFrame
 from pandas.core.generic import NDFrame
-from pandas.core.index import (Index, MultiIndex, CategoricalIndex,
-                               _ensure_index)
+from pandas.core.interval import IntervalIndex
 from pandas.core.internals import BlockManager, make_block
 from pandas.core.series import Series
 from pandas.core.panel import Panel
@@ -3134,12 +3133,20 @@ def value_counts(self, normalize=False, sort=True, ascending=False,
         if bins is None:
             lab, lev = algorithms.factorize(val, sort=True)
         else:
-            cat, bins = cut(val, bins, retbins=True)
+            raise NotImplementedError('this is broken')
+            lab, bins = cut(val, bins, retbins=True)
             # bins[:-1] for backward compat;
             # o.w. cat.categories could be better
-            lab, lev, dropna = cat.codes, bins[:-1], False
-
-        sorter = np.lexsort((lab, ids))
+            # cat = Categorical(cat)
+            # lab, lev, dropna = cat.codes, bins[:-1], False
+
+        if (lab.dtype == object
+                and lib.is_interval_array_fixed_closed(lab[notnull(lab)])):
+            lab_index = Index(lab)
+            assert isinstance(lab, IntervalIndex)
+            sorter = np.lexsort((lab_index.left, lab_index.right, ids))
+        else:
+            sorter = np.lexsort((lab, ids))
         ids, lab = ids[sorter], lab[sorter]
 
         # group boundaries are where group ids change
@@ -3180,12 +3187,13 @@ def value_counts(self, normalize=False, sort=True, ascending=False,
                 acc = rep(d)
             out /= acc
 
-        if sort and bins is None:
+        if sort:  # and bins is None:
             cat = ids[inc][mask] if dropna else ids[inc]
             sorter = np.lexsort((out if ascending else -out, cat))
             out, labels[-1] = out[sorter], labels[-1][sorter]
 
-        if bins is None:
+        # if bins is None:
+        if True:
             mi = MultiIndex(levels=levels, labels=labels, names=names,
                             verify_integrity=False)