ENH: Create and propagate UInt64Index

[ci skip]

Author: gfyoung

doc/source/whatsnew/v0.20.0.txt

@@ -88,6 +88,46 @@ support for bz2 compression in the python 2 c-engine improved (:issue:`14874`).
    df = pd.read_table(url, compression='bz2')  # explicitly specify compression
    df.head(2)
 
+.. _whatsnew_0200.enhancements.uint64_support:
+
+Pandas has significantly improved support for operations involving unsigned,
+or purely non-negative, integers. Previously, handling these integers would
+result in improper rounding or data-type casting, leading to incorrect results.
+One notable place where this improved was in ``DataFrame`` creation (:issue:`14917`):
+
+.. ipython:: python
+
+   arr = np.array([1, 2**63, 5], dtype=np.uint64)
+   arr
+
+**Previous behavior**:
+
+.. code-block:: ipython
+
+   In [3]: df = DataFrame({'a': arr})
+
+   In [4]: df['a']
+   Out[4]:
+                          a
+   0                      1
+   1    9223372036854775808
+   2                      5
+   Name: a, dtype: object
+
+The correct data type should unsigned 64-bit integer, not object. This release
+rectifies this behavior as seen below:
+
+**New behavior**:
+
+.. ipython:: python
+
+   df = DataFrame({'a': arr})
+   df['a']
+
+- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`)
+- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
+- New ``UInt64Index`` (subclass of ``NumericIndex``) for specifically indexing unsigned integers (:issue:`14935`)
+
 .. _whatsnew_0200.enhancements.other:
 
 Other enhancements
@@ -279,7 +319,6 @@ Bug Fixes
 ~~~~~~~~~
 
 - Bug in ``TimedeltaIndex`` addition where overflow was being allowed without error (:issue:`14816`)
-- Bug in ``DataFrame`` construction in which unsigned 64-bit integer elements were being converted to objects (:issue:`14881`)
 - Bug in ``astype()`` where ``inf`` values were incorrectly converted to integers. Now raises error now with ``astype()`` for Series and DataFrames (:issue:`14265`)
 - Bug in ``DataFrame(..).apply(to_numeric)`` when values are of type decimal.Decimal. (:issue:`14827`)
 - Bug in ``describe()`` when passing a numpy array which does not contain the median to the ``percentiles`` keyword argument (:issue:`14908`)
@@ -297,6 +336,4 @@ Bug Fixes
 
 
 
-- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
 - Require at least 0.23 version of cython to avoid problems with character encodings (:issue:`14699`)
-- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`)

pandas/api/tests/test_api.py

@@ -53,7 +53,7 @@ class TestPDApi(Base, tm.TestCase):
     classes = ['Categorical', 'CategoricalIndex', 'DataFrame', 'DateOffset',
                'DatetimeIndex', 'ExcelFile', 'ExcelWriter', 'Float64Index',
                'Grouper', 'HDFStore', 'Index', 'Int64Index', 'MultiIndex',
-               'Period', 'PeriodIndex', 'RangeIndex',
+               'Period', 'PeriodIndex', 'RangeIndex', 'UInt64Index',
                'Series', 'SparseArray', 'SparseDataFrame',
                'SparseSeries', 'TimeGrouper', 'Timedelta',
                'TimedeltaIndex', 'Timestamp']

pandas/core/api.py

@@ -10,7 +10,8 @@
 from pandas.core.groupby import Grouper
 from pandas.formats.format import set_eng_float_format
 from pandas.core.index import (Index, CategoricalIndex, Int64Index,
-                               RangeIndex, Float64Index, MultiIndex)
+                               UInt64Index, RangeIndex, Float64Index,
+                               MultiIndex)
 
 from pandas.core.series import Series, TimeSeries
 from pandas.core.frame import DataFrame

pandas/index.pyx

@@ -424,6 +424,67 @@ cdef class Int64Engine(IndexEngine):
 
         return result
 
+cdef class UInt64Engine(IndexEngine):
+
+    cdef _get_index_values(self):
+        return algos.ensure_uint64(self.vgetter())
+
+    cdef _make_hash_table(self, n):
+        return _hash.UInt64HashTable(n)
+
+    def _call_monotonic(self, values):
+        return algos.is_monotonic_uint64(values, timelike=False)
+
+    def get_pad_indexer(self, other, limit=None):
+        return algos.pad_uint64(self._get_index_values(), other,
+                                limit=limit)
+
+    def get_backfill_indexer(self, other, limit=None):
+        return algos.backfill_uint64(self._get_index_values(), other,
+                                     limit=limit)
+
+    cdef _check_type(self, object val):
+        hash(val)
+        if util.is_bool_object(val):
+            raise KeyError(val)
+        elif util.is_float_object(val):
+            raise KeyError(val)
+
+    cdef _maybe_get_bool_indexer(self, object val):
+        cdef:
+            ndarray[uint8_t, cast=True] indexer
+            ndarray[uint64_t] values
+            int count = 0
+            Py_ssize_t i, n
+            uint64_t uval
+            int last_true
+
+        if not util.is_integer_object(val):
+            raise KeyError(val)
+
+        uval = val
+
+        values = self._get_index_values()
+        n = len(values)
+
+        result = np.empty(n, dtype=bool)
+        indexer = result.view(np.uint8)
+
+        for i in range(n):
+            if values[i] == uval:
+                count += 1
+                indexer[i] = 1
+                last_true = i
+            else:
+                indexer[i] = 0
+
+        if count == 0:
+            raise KeyError(val)
+        if count == 1:
+            return last_true
+
+        return result
+
 cdef class Float64Engine(IndexEngine):
 
     cdef _make_hash_table(self, n):

pandas/indexes/api.py

@@ -4,7 +4,7 @@
 from pandas.indexes.category import CategoricalIndex  # noqa
 from pandas.indexes.multi import MultiIndex  # noqa
 from pandas.indexes.numeric import (NumericIndex, Float64Index,  # noqa
-                                    Int64Index)
+                                    Int64Index, UInt64Index)
 from pandas.indexes.range import RangeIndex  # noqa
 
 import pandas.core.common as com
@@ -13,7 +13,7 @@
 # TODO: there are many places that rely on these private methods existing in
 # pandas.core.index
 __all__ = ['Index', 'MultiIndex', 'NumericIndex', 'Float64Index', 'Int64Index',
-           'CategoricalIndex', 'RangeIndex',
+           'CategoricalIndex', 'RangeIndex', 'UInt64Index',
            'InvalidIndexError',
            '_new_Index',
            '_ensure_index', '_get_na_value', '_get_combined_index',

pandas/indexes/base.py

@@ -199,14 +199,25 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                             data = np.array(data, copy=copy, dtype=dtype)
                         elif inferred in ['floating', 'mixed-integer-float']:
 
-                            # if we are actually all equal to integers
+                            # If we are actually all equal to integers,
                             # then coerce to integer
-                            from .numeric import Int64Index, Float64Index
+                            from .numeric import (Int64Index, UInt64Index,
+                                                  Float64Index)
                             try:
                                 res = data.astype('i8')
                                 if (res == data).all():
                                     return Int64Index(res, copy=copy,
                                                       name=name)
+                            except (OverflowError, TypeError, ValueError):
+                                pass
+
+                            # Conversion to int64 failed (possibly due to
+                            # overflow), so let's try now with uint64.
+                            try:
+                                res = data.astype('u8')
+                                if (res == data).all():
+                                    return UInt64Index(res, copy=copy,
+                                                       name=name)
                             except (TypeError, ValueError):
                                 pass
 
@@ -235,9 +246,12 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                                                IncompatibleFrequency)
             if isinstance(data, PeriodIndex):
                 return PeriodIndex(data, copy=copy, name=name, **kwargs)
-            if issubclass(data.dtype.type, np.integer):
+            if issubclass(data.dtype.type, np.signedinteger):
                 from .numeric import Int64Index
                 return Int64Index(data, copy=copy, dtype=dtype, name=name)
+            elif issubclass(data.dtype.type, np.unsignedinteger):
+                from .numeric import UInt64Index
+                return UInt64Index(data, copy=copy, dtype=dtype, name=name)
             elif issubclass(data.dtype.type, np.floating):
                 from .numeric import Float64Index
                 return Float64Index(data, copy=copy, dtype=dtype, name=name)
@@ -254,9 +268,13 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
             if dtype is None:
                 inferred = lib.infer_dtype(subarr)
                 if inferred == 'integer':
-                    from .numeric import Int64Index
-                    return Int64Index(subarr.astype('i8'), copy=copy,
-                                      name=name)
+                    from .numeric import Int64Index, UInt64Index
+                    try:
+                        return Int64Index(subarr.astype('i8'), copy=copy,
+                                          name=name)
+                    except OverflowError:
+                        return UInt64Index(subarr.astype('u8'), copy=copy,
+                                           name=name)
                 elif inferred in ['floating', 'mixed-integer-float']:
                     from .numeric import Float64Index
                     return Float64Index(subarr, copy=copy, name=name)

pandas/indexes/numeric.py

@@ -177,6 +177,91 @@ def _assert_safe_casting(cls, data, subarr):
 Int64Index._add_logical_methods()
 
 
+class UInt64Index(NumericIndex):
+    """
+    Immutable ndarray implementing an ordered, sliceable set. The basic object
+    storing axis labels for all pandas objects. UInt64Index is a special case
+    of `Index` with purely integer labels.
+
+    Parameters
+    ----------
+    data : array-like (1-dimensional)
+    dtype : NumPy dtype (default: uint64)
+    copy : bool
+        Make a copy of input ndarray
+    name : object
+        Name to be stored in the index
+
+    Notes
+    -----
+    An Index instance can **only** contain hashable objects
+    """
+
+    _typ = 'uint64index'
+    _arrmap = _algos.arrmap_uint64
+    _left_indexer_unique = _join.left_join_indexer_unique_uint64
+    _left_indexer = _join.left_join_indexer_uint64
+    _inner_indexer = _join.inner_join_indexer_uint64
+    _outer_indexer = _join.outer_join_indexer_uint64
+
+    _can_hold_na = False
+
+    _engine_type = _index.UInt64Engine
+
+    _default_dtype = np.uint64
+
+    @property
+    def inferred_type(self):
+        return 'integer'
+
+    @property
+    def asi8(self):
+        # do not cache or you'll create a memory leak
+        return self.values.view('u8')
+
+    @property
+    def is_all_dates(self):
+        """
+        Checks that all the labels are datetime objects
+        """
+        return False
+
+    def _convert_scalar_indexer(self, key, kind=None):
+        """
+        convert a scalar indexer
+
+        Parameters
+        ----------
+        key : label of the slice bound
+        kind : {'ix', 'loc', 'getitem'} or None
+        """
+
+        assert kind in ['ix', 'loc', 'getitem', 'iloc', None]
+
+        # don't coerce ilocs to integers
+        if kind != 'iloc':
+            key = self._maybe_cast_indexer(key)
+        return (super(UInt64Index, self)
+                ._convert_scalar_indexer(key, kind=kind))
+
+    def _wrap_joined_index(self, joined, other):
+        name = self.name if self.name == other.name else None
+        return Int64Index(joined, name=name)
+
+    @classmethod
+    def _assert_safe_casting(cls, data, subarr):
+        """
+        Ensure incoming data can be represented as uints.
+        """
+        if not issubclass(data.dtype.type, np.integer):
+            if not np.array_equal(data, subarr):
+                raise TypeError('Unsafe NumPy casting, you must '
+                                'explicitly cast')
+
+UInt64Index._add_numeric_methods()
+UInt64Index._add_logical_methods()
+
+
 class Float64Index(NumericIndex):
     """
     Immutable ndarray implementing an ordered, sliceable set. The basic object

pandas/src/algos_common_helper.pxi.in

@@ -27,6 +27,7 @@ dtypes = [('float64', 'float64_t', 'np.float64', True, True),
           ('object', 'object', 'object', True, False),
           ('int32', 'int32_t', 'np.int32', False, True),
           ('int64', 'int64_t', 'np.int64', False, True),
+          ('uint64', 'uint64_t', 'np.uint64', False, True),
           ('bool', 'uint8_t', 'np.bool', False, True)]
 
 def get_dispatch(dtypes):

pandas/src/inference.pyx

@@ -810,7 +810,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
             floats[i] = <float64_t> val
             complexes[i] = <double complex> val
             if not seen_null:
-                seen_uint = seen_uint or (val > npy_int64_max)
+                seen_uint = seen_uint or (int(val) > npy_int64_max)
                 seen_sint = seen_sint or (val < 0)
 
                 if seen_uint and seen_sint:

pandas/src/join_helper.pxi.in

@@ -15,7 +15,8 @@ dtypes = [('float64', 'float64_t', 'np.float64'),
           ('float32', 'float32_t', 'np.float32'),
           ('object', 'object', 'object'),
           ('int32', 'int32_t', 'np.int32'),
-          ('int64', 'int64_t', 'np.int64')]
+          ('int64', 'int64_t', 'np.int64'),
+          ('uint64', 'uint64_t', 'np.uint64')]
 
 def get_dispatch(dtypes):
 
@@ -404,4 +405,4 @@ def outer_join_indexer_{{name}}(ndarray[{{c_type}}] left,
 
     return result, lindexer, rindexer
 
-{{endfor}}
+{{endfor}}

pandas/src/joins_func_helper.pxi.in

@@ -12,7 +12,8 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 {{py:
 
 # table_type, by_dtype
-by_dtypes = [('PyObjectHashTable', 'object'), ('Int64HashTable', 'int64_t')]
+by_dtypes = [('PyObjectHashTable', 'object'), ('Int64HashTable', 'int64_t'),
+             ('UInt64HashTable', 'uint64_t')]
 
 # on_dtype
 on_dtypes = ['uint8_t', 'uint16_t', 'uint32_t', 'uint64_t',