ENH: Create and propagate UInt64Index

[ci skip]

Author: gfyoung

doc/source/whatsnew/v0.20.0.txt

@@ -88,6 +88,46 @@ support for bz2 compression in the python 2 c-engine improved (:issue:`14874`).
    df = pd.read_table(url, compression='bz2')  # explicitly specify compression
    df.head(2)
 
+.. _whatsnew_0200.enhancements.uint64_support:
+
+Pandas has significantly improved support for operations involving unsigned,
+or purely non-negative, integers. Previously, handling these integers would
+result in improper rounding or data-type casting, leading to incorrect results.
+One notable place where this improved was in ``DataFrame`` creation (:issue:`14917`):
+
+.. ipython:: python
+
+   arr = np.array([1, 2**63, 5], dtype=np.uint64)
+   arr
+
+**Previous behavior**:
+
+.. code-block:: ipython
+
+   In [3]: df = DataFrame({'a': arr})
+
+   In [4]: df['a']
+   Out[4]:
+                          a
+   0                      1
+   1    9223372036854775808
+   2                      5
+   Name: a, dtype: object
+
+The correct data type should unsigned 64-bit integer, not object. This release
+rectifies this behavior as seen below:
+
+**New behavior**:
+
+.. ipython:: python
+
+   df = DataFrame({'a': arr})
+   df['a']
+
+- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`)
+- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
+- New ``UInt64Index`` (subclass of ``NumericIndex``) for specifically indexing unsigned integers (:issue:`14935`)
+
 .. _whatsnew_0200.enhancements.other:
 
 Other enhancements
@@ -279,7 +319,6 @@ Bug Fixes
 ~~~~~~~~~
 
 - Bug in ``TimedeltaIndex`` addition where overflow was being allowed without error (:issue:`14816`)
-- Bug in ``DataFrame`` construction in which unsigned 64-bit integer elements were being converted to objects (:issue:`14881`)
 - Bug in ``astype()`` where ``inf`` values were incorrectly converted to integers. Now raises error now with ``astype()`` for Series and DataFrames (:issue:`14265`)
 - Bug in ``DataFrame(..).apply(to_numeric)`` when values are of type decimal.Decimal. (:issue:`14827`)
 - Bug in ``describe()`` when passing a numpy array which does not contain the median to the ``percentiles`` keyword argument (:issue:`14908`)
@@ -297,6 +336,4 @@ Bug Fixes
 
 
 
-- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
 - Require at least 0.23 version of cython to avoid problems with character encodings (:issue:`14699`)
-- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`)

pandas/api/tests/test_api.py

@@ -53,7 +53,7 @@ class TestPDApi(Base, tm.TestCase):
     classes = ['Categorical', 'CategoricalIndex', 'DataFrame', 'DateOffset',
                'DatetimeIndex', 'ExcelFile', 'ExcelWriter', 'Float64Index',
                'Grouper', 'HDFStore', 'Index', 'Int64Index', 'MultiIndex',
-               'Period', 'PeriodIndex', 'RangeIndex',
+               'Period', 'PeriodIndex', 'RangeIndex', 'UInt64Index',
                'Series', 'SparseArray', 'SparseDataFrame',
                'SparseSeries', 'TimeGrouper', 'Timedelta',
                'TimedeltaIndex', 'Timestamp']

pandas/core/api.py

@@ -10,7 +10,8 @@
 from pandas.core.groupby import Grouper
 from pandas.formats.format import set_eng_float_format
 from pandas.core.index import (Index, CategoricalIndex, Int64Index,
-                               RangeIndex, Float64Index, MultiIndex)
+                               UInt64Index, RangeIndex, Float64Index,
+                               MultiIndex)
 
 from pandas.core.series import Series, TimeSeries
 from pandas.core.frame import DataFrame

pandas/index.pyx

@@ -424,6 +424,67 @@ cdef class Int64Engine(IndexEngine):
 
         return result
 
+cdef class UInt64Engine(IndexEngine):
+
+    cdef _get_index_values(self):
+        return algos.ensure_uint64(self.vgetter())
+
+    cdef _make_hash_table(self, n):
+        return _hash.UInt64HashTable(n)
+
+    def _call_monotonic(self, values):
+        return algos.is_monotonic_uint64(values, timelike=False)
+
+    def get_pad_indexer(self, other, limit=None):
+        return algos.pad_uint64(self._get_index_values(), other,
+                                limit=limit)
+
+    def get_backfill_indexer(self, other, limit=None):
+        return algos.backfill_uint64(self._get_index_values(), other,
+                                     limit=limit)
+
+    cdef _check_type(self, object val):
+        hash(val)
+        if util.is_bool_object(val):
+            raise KeyError(val)
+        elif util.is_float_object(val):
+            raise KeyError(val)
+
+    cdef _maybe_get_bool_indexer(self, object val):
+        cdef:
+            ndarray[uint8_t, cast=True] indexer
+            ndarray[uint64_t] values
+            int count = 0
+            Py_ssize_t i, n
+            uint64_t uval
+            int last_true
+
+        if not util.is_integer_object(val):
+            raise KeyError(val)
+
+        uval = val
+
+        values = self._get_index_values()
+        n = len(values)
+
+        result = np.empty(n, dtype=bool)
+        indexer = result.view(np.uint8)
+
+        for i in range(n):
+            if values[i] == uval:
+                count += 1
+                indexer[i] = 1
+                last_true = i
+            else:
+                indexer[i] = 0
+
+        if count == 0:
+            raise KeyError(val)
+        if count == 1:
+            return last_true
+
+        return result
+
 cdef class Float64Engine(IndexEngine):
 
     cdef _make_hash_table(self, n):

pandas/indexes/api.py

@@ -4,7 +4,7 @@
 from pandas.indexes.category import CategoricalIndex  # noqa
 from pandas.indexes.multi import MultiIndex  # noqa
 from pandas.indexes.numeric import (NumericIndex, Float64Index,  # noqa
-                                    Int64Index)
+                                    Int64Index, UInt64Index)
 from pandas.indexes.range import RangeIndex  # noqa
 
 import pandas.core.common as com
@@ -13,7 +13,7 @@
 # TODO: there are many places that rely on these private methods existing in
 # pandas.core.index
 __all__ = ['Index', 'MultiIndex', 'NumericIndex', 'Float64Index', 'Int64Index',
-           'CategoricalIndex', 'RangeIndex',
+           'CategoricalIndex', 'RangeIndex', 'UInt64Index',
            'InvalidIndexError',
            '_new_Index',
            '_ensure_index', '_get_na_value', '_get_combined_index',

pandas/indexes/base.py

@@ -199,14 +199,25 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                             data = np.array(data, copy=copy, dtype=dtype)
                         elif inferred in ['floating', 'mixed-integer-float']:
 
-                            # if we are actually all equal to integers
+                            # If we are actually all equal to integers,
                             # then coerce to integer
-                            from .numeric import Int64Index, Float64Index
+                            from .numeric import (Int64Index, UInt64Index,
+                                                  Float64Index)
                             try:
                                 res = data.astype('i8')
                                 if (res == data).all():
                                     return Int64Index(res, copy=copy,
                                                       name=name)
+                            except (OverflowError, TypeError, ValueError):
+                                pass
+
+                            # Conversion to int64 failed (possibly due to
+                            # overflow), so let's try now with uint64.
+                            try:
+                                res = data.astype('u8')
+                                if (res == data).all():
+                                    return UInt64Index(res, copy=copy,
+                                                       name=name)
                             except (TypeError, ValueError):
                                 pass
 
@@ -235,9 +246,12 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                                                IncompatibleFrequency)
             if isinstance(data, PeriodIndex):
                 return PeriodIndex(data, copy=copy, name=name, **kwargs)
-            if issubclass(data.dtype.type, np.integer):
+            if issubclass(data.dtype.type, np.signedinteger):
                 from .numeric import Int64Index
                 return Int64Index(data, copy=copy, dtype=dtype, name=name)
+            elif issubclass(data.dtype.type, np.unsignedinteger):
+                from .numeric import UInt64Index
+                return UInt64Index(data, copy=copy, dtype=dtype, name=name)
             elif issubclass(data.dtype.type, np.floating):
                 from .numeric import Float64Index
                 return Float64Index(data, copy=copy, dtype=dtype, name=name)
@@ -254,9 +268,13 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
             if dtype is None:
                 inferred = lib.infer_dtype(subarr)
                 if inferred == 'integer':
-                    from .numeric import Int64Index
-                    return Int64Index(subarr.astype('i8'), copy=copy,
-                                      name=name)
+                    from .numeric import Int64Index, UInt64Index
+                    try:
+                        return Int64Index(subarr.astype('i8'), copy=copy,
+                                          name=name)
+                    except OverflowError:
+                        return UInt64Index(subarr.astype('u8'), copy=copy,
+                                           name=name)
                 elif inferred in ['floating', 'mixed-integer-float']:
                     from .numeric import Float64Index
                     return Float64Index(subarr, copy=copy, name=name)