ENH: Create and propagate UInt64Index

Author: gfyoung

doc/source/whatsnew/v0.20.0.txt

@@ -99,6 +99,7 @@ Other enhancements
   unsorted MultiIndex (:issue:`11897`). This allows differentiation between errors due to lack
   of sorting or an incorrect key. See :ref:`here <advanced.unsorted>`
 
+- New ``UInt64Index`` (subclass of ``NumericIndex``) for specifically indexing unsigned integers (:issue:`14935`)
 - ``pd.cut`` and ``pd.qcut`` now support datetime64 and timedelta64 dtypes (:issue:`14714`)
 - ``Series`` provides a ``to_excel`` method to output Excel files (:issue:`8825`)
 - The ``usecols`` argument in ``pd.read_csv`` now accepts a callable function as a value  (:issue:`14154`)

pandas/api/tests/test_api.py

@@ -53,7 +53,7 @@ class TestPDApi(Base, tm.TestCase):
     classes = ['Categorical', 'CategoricalIndex', 'DataFrame', 'DateOffset',
                'DatetimeIndex', 'ExcelFile', 'ExcelWriter', 'Float64Index',
                'Grouper', 'HDFStore', 'Index', 'Int64Index', 'MultiIndex',
-               'Period', 'PeriodIndex', 'RangeIndex',
+               'Period', 'PeriodIndex', 'RangeIndex', 'UInt64Index',
                'Series', 'SparseArray', 'SparseDataFrame',
                'SparseSeries', 'TimeGrouper', 'Timedelta',
                'TimedeltaIndex', 'Timestamp']

pandas/core/api.py

@@ -10,7 +10,8 @@
 from pandas.core.groupby import Grouper
 from pandas.formats.format import set_eng_float_format
 from pandas.core.index import (Index, CategoricalIndex, Int64Index,
-                               RangeIndex, Float64Index, MultiIndex)
+                               UInt64Index, RangeIndex, Float64Index,
+                               MultiIndex)
 
 from pandas.core.series import Series, TimeSeries
 from pandas.core.frame import DataFrame

pandas/index.pyx

@@ -426,6 +426,67 @@ cdef class Int64Engine(IndexEngine):
 
         return result
 
+cdef class UInt64Engine(IndexEngine):
+
+    cdef _get_index_values(self):
+        return algos.ensure_uint64(self.vgetter())
+
+    cdef _make_hash_table(self, n):
+        return _hash.UInt64HashTable(n)
+
+    def _call_monotonic(self, values):
+        return algos.is_monotonic_uint64(values, timelike=False)
+
+    def get_pad_indexer(self, other, limit=None):
+        return algos.pad_uint64(self._get_index_values(), other,
+                                limit=limit)
+
+    def get_backfill_indexer(self, other, limit=None):
+        return algos.backfill_uint64(self._get_index_values(), other,
+                                     limit=limit)
+
+    cdef _check_type(self, object val):
+        hash(val)
+        if util.is_bool_object(val):
+            raise KeyError(val)
+        elif util.is_float_object(val):
+            raise KeyError(val)
+
+    cdef _maybe_get_bool_indexer(self, object val):
+        cdef:
+            ndarray[uint8_t, cast=True] indexer
+            ndarray[uint64_t] values
+            int count = 0
+            Py_ssize_t i, n
+            uint64_t uval
+            int last_true
+
+        if not util.is_integer_object(val):
+            raise KeyError(val)
+
+        uval = val
+
+        values = self._get_index_values()
+        n = len(values)
+
+        result = np.empty(n, dtype=bool)
+        indexer = result.view(np.uint8)
+
+        for i in range(n):
+            if values[i] == uval:
+                count += 1
+                indexer[i] = 1
+                last_true = i
+            else:
+                indexer[i] = 0
+
+        if count == 0:
+            raise KeyError(val)
+        if count == 1:
+            return last_true
+
+        return result
+
 cdef class Float64Engine(IndexEngine):
 
     cdef _make_hash_table(self, n):

pandas/indexes/api.py

@@ -4,7 +4,7 @@
 from pandas.indexes.category import CategoricalIndex  # noqa
 from pandas.indexes.multi import MultiIndex  # noqa
 from pandas.indexes.numeric import (NumericIndex, Float64Index,  # noqa
-                                    Int64Index)
+                                    Int64Index, UInt64Index)
 from pandas.indexes.range import RangeIndex  # noqa
 
 import pandas.core.common as com
@@ -13,7 +13,7 @@
 # TODO: there are many places that rely on these private methods existing in
 # pandas.core.index
 __all__ = ['Index', 'MultiIndex', 'NumericIndex', 'Float64Index', 'Int64Index',
-           'CategoricalIndex', 'RangeIndex',
+           'CategoricalIndex', 'RangeIndex', 'UInt64Index',
            'InvalidIndexError',
            '_new_Index',
            '_ensure_index', '_get_na_value', '_get_combined_index',

pandas/indexes/base.py

@@ -201,12 +201,21 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
 
                             # if we are actually all equal to integers
                             # then coerce to integer
-                            from .numeric import Int64Index, Float64Index
+                            from .numeric import (Int64Index, UInt64Index,
+                                                  Float64Index)
                             try:
                                 res = data.astype('i8')
                                 if (res == data).all():
                                     return Int64Index(res, copy=copy,
                                                       name=name)
+                            except (OverflowError, TypeError, ValueError):
+                                pass
+
+                            try:
+                                res = data.astype('u8')
+                                if (res == data).all():
+                                    return UInt64Index(res, copy=copy,
+                                                       name=name)
                             except (TypeError, ValueError):
                                 pass
 
@@ -235,9 +244,12 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                                                IncompatibleFrequency)
             if isinstance(data, PeriodIndex):
                 return PeriodIndex(data, copy=copy, name=name, **kwargs)
-            if issubclass(data.dtype.type, np.integer):
+            if issubclass(data.dtype.type, np.signedinteger):
                 from .numeric import Int64Index
                 return Int64Index(data, copy=copy, dtype=dtype, name=name)
+            elif issubclass(data.dtype.type, np.unsignedinteger):
+                from .numeric import UInt64Index
+                return UInt64Index(data, copy=copy, dtype=dtype, name=name)
             elif issubclass(data.dtype.type, np.floating):
                 from .numeric import Float64Index
                 return Float64Index(data, copy=copy, dtype=dtype, name=name)
@@ -254,9 +266,13 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
             if dtype is None:
                 inferred = lib.infer_dtype(subarr)
                 if inferred == 'integer':
-                    from .numeric import Int64Index
-                    return Int64Index(subarr.astype('i8'), copy=copy,
-                                      name=name)
+                    from .numeric import Int64Index, UInt64Index
+                    try:
+                        return Int64Index(subarr.astype('i8'), copy=copy,
+                                          name=name)
+                    except OverflowError:
+                        return UInt64Index(subarr.astype('u8'), copy=copy,
+                                           name=name)
                 elif inferred in ['floating', 'mixed-integer-float']:
                     from .numeric import Float64Index
                     return Float64Index(subarr, copy=copy, name=name)

pandas/indexes/numeric.py

@@ -177,6 +177,91 @@ def _assert_safe_casting(cls, data, subarr):
 Int64Index._add_logical_methods()
 
 
+class UInt64Index(NumericIndex):
+    """
+    Immutable ndarray implementing an ordered, sliceable set. The basic object
+    storing axis labels for all pandas objects. UInt64Index is a special case
+    of `Index` with purely integer labels.
+
+    Parameters
+    ----------
+    data : array-like (1-dimensional)
+    dtype : NumPy dtype (default: uint64)
+    copy : bool
+        Make a copy of input ndarray
+    name : object
+        Name to be stored in the index
+
+    Notes
+    -----
+    An Index instance can **only** contain hashable objects
+    """
+
+    _typ = 'uint64index'
+    _arrmap = _algos.arrmap_uint64
+    _left_indexer_unique = _join.left_join_indexer_unique_uint64
+    _left_indexer = _join.left_join_indexer_uint64
+    _inner_indexer = _join.inner_join_indexer_uint64
+    _outer_indexer = _join.outer_join_indexer_uint64
+
+    _can_hold_na = False
+
+    _engine_type = _index.UInt64Engine
+
+    _default_dtype = np.uint64
+
+    @property
+    def inferred_type(self):
+        return 'integer'
+
+    @property
+    def asi8(self):
+        # do not cache or you'll create a memory leak
+        return self.values.view('u8')
+
+    @property
+    def is_all_dates(self):
+        """
+        Checks that all the labels are datetime objects
+        """
+        return False
+
+    def _convert_scalar_indexer(self, key, kind=None):
+        """
+        convert a scalar indexer
+
+        Parameters
+        ----------
+        key : label of the slice bound
+        kind : {'ix', 'loc', 'getitem'} or None
+        """
+
+        assert kind in ['ix', 'loc', 'getitem', 'iloc', None]
+
+        # don't coerce ilocs to integers
+        if kind != 'iloc':
+            key = self._maybe_cast_indexer(key)
+        return (super(UInt64Index, self)
+                ._convert_scalar_indexer(key, kind=kind))
+
+    def _wrap_joined_index(self, joined, other):
+        name = self.name if self.name == other.name else None
+        return Int64Index(joined, name=name)
+
+    @classmethod
+    def _assert_safe_casting(cls, data, subarr):
+        """
+        Ensure incoming data can be represented as uints.
+        """
+        if not issubclass(data.dtype.type, np.integer):
+            if not np.array_equal(data, subarr):
+                raise TypeError('Unsafe NumPy casting, you must '
+                                'explicitly cast')
+
+UInt64Index._add_numeric_methods()
+UInt64Index._add_logical_methods()
+
+
 class Float64Index(NumericIndex):
     """
     Immutable ndarray implementing an ordered, sliceable set. The basic object

pandas/src/algos_common_helper.pxi.in

@@ -27,6 +27,7 @@ dtypes = [('float64', 'float64_t', 'np.float64', True, True),
           ('object', 'object', 'object', True, False),
           ('int32', 'int32_t', 'np.int32', False, True),
           ('int64', 'int64_t', 'np.int64', False, True),
+          ('uint64', 'uint64_t', 'np.uint64', False, True),
           ('bool', 'uint8_t', 'np.bool', False, True)]
 
 def get_dispatch(dtypes):

pandas/src/inference.pyx

@@ -810,7 +810,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
             floats[i] = <float64_t> val
             complexes[i] = <double complex> val
             if not seen_null:
-                seen_uint = seen_uint or (val > npy_int64_max)
+                seen_uint = seen_uint or (int(val) > npy_int64_max)
                 seen_sint = seen_sint or (val < 0)
 
                 if seen_uint and seen_sint:

pandas/src/join_helper.pxi.in

@@ -15,7 +15,8 @@ dtypes = [('float64', 'float64_t', 'np.float64'),
           ('float32', 'float32_t', 'np.float32'),
           ('object', 'object', 'object'),
           ('int32', 'int32_t', 'np.int32'),
-          ('int64', 'int64_t', 'np.int64')]
+          ('int64', 'int64_t', 'np.int64'),
+          ('uint64', 'uint64_t', 'np.uint64')]
 
 def get_dispatch(dtypes):
 
@@ -404,4 +405,4 @@ def outer_join_indexer_{{name}}(ndarray[{{c_type}}] left,
 
     return result, lindexer, rindexer
 
-{{endfor}}
+{{endfor}}

pandas/src/joins_func_helper.pxi.in

@@ -12,7 +12,8 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 {{py:
 
 # table_type, by_dtype
-by_dtypes = [('PyObjectHashTable', 'object'), ('Int64HashTable', 'int64_t')]
+by_dtypes = [('PyObjectHashTable', 'object'), ('Int64HashTable', 'int64_t'),
+             ('UInt64HashTable', 'uint64_t')]
 
 # on_dtype
 on_dtypes = ['uint8_t', 'uint16_t', 'uint32_t', 'uint64_t',

pandas/tests/indexes/common.py

@@ -5,8 +5,8 @@
 
 import numpy as np
 
-from pandas import (Series, Index, Float64Index, Int64Index, RangeIndex,
-                    MultiIndex, CategoricalIndex, DatetimeIndex,
+from pandas import (Series, Index, Float64Index, Int64Index, UInt64Index,
+                    RangeIndex, MultiIndex, CategoricalIndex, DatetimeIndex,
                     TimedeltaIndex, PeriodIndex, notnull)
 from pandas.types.common import needs_i8_conversion
 from pandas.util.testing import assertRaisesRegexp
@@ -744,7 +744,7 @@ def test_numpy_ufuncs(self):
                     with tm.assertRaises(Exception):
                         with np.errstate(all='ignore'):
                             func(idx)
-                elif isinstance(idx, (Float64Index, Int64Index)):
+                elif isinstance(idx, (Float64Index, Int64Index, UInt64Index)):
                     # coerces to float (e.g. np.sin)
                     with np.errstate(all='ignore'):
                         result = func(idx)
@@ -765,7 +765,7 @@ def test_numpy_ufuncs(self):
                     # raise TypeError or ValueError (PeriodIndex)
                     with tm.assertRaises(Exception):
                         func(idx)
-                elif isinstance(idx, (Float64Index, Int64Index)):
+                elif isinstance(idx, (Float64Index, Int64Index, UInt64Index)):
                     # results in bool array
                     result = func(idx)
                     exp = func(idx.values)
@@ -798,7 +798,7 @@ def test_hasnans_isnans(self):
                     continue
                 elif isinstance(index, pd.tseries.base.DatetimeIndexOpsMixin):
                     values[1] = pd.tslib.iNaT
-                elif isinstance(index, Int64Index):
+                elif isinstance(index, (Int64Index, UInt64Index)):
                     continue
                 else:
                     values[1] = np.nan
@@ -838,7 +838,7 @@ def test_fillna(self):
 
                 if isinstance(index, pd.tseries.base.DatetimeIndexOpsMixin):
                     values[1] = pd.tslib.iNaT
-                elif isinstance(index, Int64Index):
+                elif isinstance(index, (Int64Index, UInt64Index)):
                     continue
                 else:
                     values[1] = np.nan

pandas/tests/indexes/test_base.py

@@ -40,6 +40,7 @@ def setUp(self):
                             periodIndex=tm.makePeriodIndex(100),
                             tdIndex=tm.makeTimedeltaIndex(100),
                             intIndex=tm.makeIntIndex(100),
+                            uintIndex=tm.makeUIntIndex(100),
                             rangeIndex=tm.makeIntIndex(100),
                             floatIndex=tm.makeFloatIndex(100),
                             boolIndex=Index([True, False]),
@@ -363,6 +364,19 @@ def test_constructor_dtypes_timedelta(self):
                         pd.TimedeltaIndex(list(values), dtype=dtype)]:
                 tm.assert_index_equal(res, idx)
 
+    def test_constructor_uint64(self):
+        idx = pd.UInt64Index([1, 2, 3])
+        res = pd.Index([1, 2, 3], dtype=np.uint64)
+        tm.assert_index_equal(res, idx)
+
+        idx = pd.UInt64Index([1, 2**63])
+        res = pd.Index([1, 2**63], dtype=np.uint64)
+        tm.assert_index_equal(res, idx)
+
+        idx = pd.UInt64Index([1, 2**63])
+        res = pd.Index([1, 2**63])
+        tm.assert_index_equal(res, idx)
+
     def test_view_with_args(self):
 
         restricted = ['unicodeIndex', 'strIndex', 'catIndex', 'boolIndex',

pandas/tests/types/test_generic.py

@@ -24,6 +24,7 @@ class TestABCClasses(tm.TestCase):
     def test_abc_types(self):
         self.assertIsInstance(pd.Index(['a', 'b', 'c']), gt.ABCIndex)
         self.assertIsInstance(pd.Int64Index([1, 2, 3]), gt.ABCInt64Index)
+        self.assertIsInstance(pd.UInt64Index([1, 2, 3]), gt.ABCUInt64Index)
         self.assertIsInstance(pd.Float64Index([1, 2, 3]), gt.ABCFloat64Index)
         self.assertIsInstance(self.multi_index, gt.ABCMultiIndex)
         self.assertIsInstance(self.datetime_index, gt.ABCDatetimeIndex)

pandas/tests/types/test_inference.py

@@ -260,6 +260,13 @@ def test_maybe_convert_objects_uint64(self):
         exp = np.array([2**63], dtype=np.uint64)
         tm.assert_numpy_array_equal(lib.maybe_convert_objects(arr), exp)
 
+        # NumPy bug: can't compare uint64 to int64, as that
+        # results in both casting to float64, so we should
+        # make sure that this function is robust against it
+        arr = np.array([np.uint64(2**63)], dtype=object)
+        exp = np.array([2**63], dtype=np.uint64)
+        tm.assert_numpy_array_equal(lib.maybe_convert_objects(arr), exp)
+
         arr = np.array([2, -1], dtype=object)
         exp = np.array([2, -1], dtype=np.int64)
         tm.assert_numpy_array_equal(lib.maybe_convert_objects(arr), exp)