ENH: Create and propagate UInt64Index

Author: gfyoung

doc/source/whatsnew/v0.20.0.txt

@@ -88,6 +88,23 @@ support for bz2 compression in the python 2 c-engine improved (:issue:`14874`).
    df = pd.read_table(url, compression='bz2')  # explicitly specify compression
    df.head(2)
 
+.. _whatsnew_0200.enhancements.uint64_support:
+
+Pandas has significantly improved support for operations involving unsigned,
+or purely non-negative, integers. Previously, handling these integers would
+result in improper rounding or data-type casting, leading to incorrect results.
+Notably, a new numerical index, `UInt64Index`, has been created (:issue:`14937`)
+
+.. ipython:: python
+
+   idx = pd.UInt64Index([1, 2, 3])
+   df = pd.DataFrame(['a', 'b', 'c'], index=idx)
+   df.index
+
+- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`)
+- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
+- Bug in ``DataFrame`` construction in which unsigned 64-bit integer elements were being converted to objects (:issue:`14881`)
+
 .. _whatsnew_0200.enhancements.other:
 
 Other enhancements
@@ -279,7 +296,6 @@ Bug Fixes
 ~~~~~~~~~
 
 - Bug in ``TimedeltaIndex`` addition where overflow was being allowed without error (:issue:`14816`)
-- Bug in ``DataFrame`` construction in which unsigned 64-bit integer elements were being converted to objects (:issue:`14881`)
 - Bug in ``astype()`` where ``inf`` values were incorrectly converted to integers. Now raises error now with ``astype()`` for Series and DataFrames (:issue:`14265`)
 - Bug in ``DataFrame(..).apply(to_numeric)`` when values are of type decimal.Decimal. (:issue:`14827`)
 - Bug in ``describe()`` when passing a numpy array which does not contain the median to the ``percentiles`` keyword argument (:issue:`14908`)
@@ -297,6 +313,4 @@ Bug Fixes
 
 
 
-- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
 - Require at least 0.23 version of cython to avoid problems with character encodings (:issue:`14699`)
-- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`)

pandas/api/tests/test_api.py

@@ -53,7 +53,7 @@ class TestPDApi(Base, tm.TestCase):
     classes = ['Categorical', 'CategoricalIndex', 'DataFrame', 'DateOffset',
                'DatetimeIndex', 'ExcelFile', 'ExcelWriter', 'Float64Index',
                'Grouper', 'HDFStore', 'Index', 'Int64Index', 'MultiIndex',
-               'Period', 'PeriodIndex', 'RangeIndex',
+               'Period', 'PeriodIndex', 'RangeIndex', 'UInt64Index',
                'Series', 'SparseArray', 'SparseDataFrame',
                'SparseSeries', 'TimeGrouper', 'Timedelta',
                'TimedeltaIndex', 'Timestamp']

pandas/core/api.py

@@ -10,7 +10,8 @@
 from pandas.core.groupby import Grouper
 from pandas.formats.format import set_eng_float_format
 from pandas.core.index import (Index, CategoricalIndex, Int64Index,
-                               RangeIndex, Float64Index, MultiIndex)
+                               UInt64Index, RangeIndex, Float64Index,
+                               MultiIndex)
 
 from pandas.core.series import Series, TimeSeries
 from pandas.core.frame import DataFrame

pandas/index.pyx

@@ -363,115 +363,6 @@ cdef class IndexEngine:
 
         return result[0:count], missing[0:count_missing]
 
-cdef class Int64Engine(IndexEngine):
-
-    cdef _get_index_values(self):
-        return algos.ensure_int64(self.vgetter())
-
-    cdef _make_hash_table(self, n):
-        return _hash.Int64HashTable(n)
-
-    def _call_monotonic(self, values):
-        return algos.is_monotonic_int64(values, timelike=False)
-
-    def get_pad_indexer(self, other, limit=None):
-        return algos.pad_int64(self._get_index_values(), other,
-                               limit=limit)
-
-    def get_backfill_indexer(self, other, limit=None):
-        return algos.backfill_int64(self._get_index_values(), other,
-                                    limit=limit)
-
-    cdef _check_type(self, object val):
-        hash(val)
-        if util.is_bool_object(val):
-            raise KeyError(val)
-        elif util.is_float_object(val):
-            raise KeyError(val)
-
-    cdef _maybe_get_bool_indexer(self, object val):
-        cdef:
-            ndarray[uint8_t, cast=True] indexer
-            ndarray[int64_t] values
-            int count = 0
-            Py_ssize_t i, n
-            int64_t ival
-            int last_true
-
-        if not util.is_integer_object(val):
-            raise KeyError(val)
-
-        ival = val
-
-        values = self._get_index_values()
-        n = len(values)
-
-        result = np.empty(n, dtype=bool)
-        indexer = result.view(np.uint8)
-
-        for i in range(n):
-            if values[i] == val:
-                count += 1
-                indexer[i] = 1
-                last_true = i
-            else:
-                indexer[i] = 0
-
-        if count == 0:
-            raise KeyError(val)
-        if count == 1:
-            return last_true
-
-        return result
-
-cdef class Float64Engine(IndexEngine):
-
-    cdef _make_hash_table(self, n):
-        return _hash.Float64HashTable(n)
-
-    cdef _get_index_values(self):
-        return algos.ensure_float64(self.vgetter())
-
-    cdef _maybe_get_bool_indexer(self, object val):
-        cdef:
-            ndarray[uint8_t] indexer
-            ndarray[float64_t] values
-            int count = 0
-            Py_ssize_t i, n
-            int last_true
-
-        values = self._get_index_values()
-        n = len(values)
-
-        result = np.empty(n, dtype=bool)
-        indexer = result.view(np.uint8)
-
-        for i in range(n):
-            if values[i] == val:
-                count += 1
-                indexer[i] = 1
-                last_true = i
-            else:
-                indexer[i] = 0
-
-        if count == 0:
-            raise KeyError(val)
-        if count == 1:
-            return last_true
-
-        return result
-
-    def _call_monotonic(self, values):
-        return algos.is_monotonic_float64(values, timelike=False)
-
-    def get_pad_indexer(self, other, limit=None):
-        return algos.pad_float64(self._get_index_values(), other,
-                                    limit=limit)
-
-    def get_backfill_indexer(self, other, limit=None):
-        return algos.backfill_float64(self._get_index_values(), other,
-                                         limit=limit)
-
 
 cdef Py_ssize_t _bin_search(ndarray values, object val) except -1:
     cdef:
@@ -510,22 +401,6 @@ _backfill_functions = {
     'float64': algos.backfill_float64
 }
 
-cdef class ObjectEngine(IndexEngine):
-
-    cdef _make_hash_table(self, n):
-        return _hash.PyObjectHashTable(n)
-
-    def _call_monotonic(self, values):
-        return algos.is_monotonic_object(values, timelike=False)
-
-    def get_pad_indexer(self, other, limit=None):
-        return algos.pad_object(self._get_index_values(), other,
-                                   limit=limit)
-
-    def get_backfill_indexer(self, other, limit=None):
-        return algos.backfill_object(self._get_index_values(), other,
-                                        limit=limit)
-
 
 cdef class DatetimeEngine(Int64Engine):
 
@@ -668,3 +543,7 @@ cdef inline _to_i8(object val):
 
 cdef inline bint _is_utc(object tz):
     return tz is UTC or isinstance(tz, _du_utc)
+
+
+# Generated from template.
+include "index_class_helper.pxi"

pandas/indexes/api.py

@@ -4,7 +4,7 @@
 from pandas.indexes.category import CategoricalIndex  # noqa
 from pandas.indexes.multi import MultiIndex  # noqa
 from pandas.indexes.numeric import (NumericIndex, Float64Index,  # noqa
-                                    Int64Index)
+                                    Int64Index, UInt64Index)
 from pandas.indexes.range import RangeIndex  # noqa
 
 import pandas.core.common as com
@@ -13,7 +13,7 @@
 # TODO: there are many places that rely on these private methods existing in
 # pandas.core.index
 __all__ = ['Index', 'MultiIndex', 'NumericIndex', 'Float64Index', 'Int64Index',
-           'CategoricalIndex', 'RangeIndex',
+           'CategoricalIndex', 'RangeIndex', 'UInt64Index',
            'InvalidIndexError',
            '_new_Index',
            '_ensure_index', '_get_na_value', '_get_combined_index',

pandas/indexes/base.py

@@ -199,14 +199,25 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                             data = np.array(data, copy=copy, dtype=dtype)
                         elif inferred in ['floating', 'mixed-integer-float']:
 
-                            # if we are actually all equal to integers
+                            # If we are actually all equal to integers,
                             # then coerce to integer
-                            from .numeric import Int64Index, Float64Index
+                            from .numeric import (Int64Index, UInt64Index,
+                                                  Float64Index)
                             try:
                                 res = data.astype('i8')
                                 if (res == data).all():
                                     return Int64Index(res, copy=copy,
                                                       name=name)
+                            except (OverflowError, TypeError, ValueError):
+                                pass
+
+                            # Conversion to int64 failed (possibly due to
+                            # overflow), so let's try now with uint64.
+                            try:
+                                res = data.astype('u8')
+                                if (res == data).all():
+                                    return UInt64Index(res, copy=copy,
+                                                       name=name)
                             except (TypeError, ValueError):
                                 pass
 
@@ -235,9 +246,12 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                                                IncompatibleFrequency)
             if isinstance(data, PeriodIndex):
                 return PeriodIndex(data, copy=copy, name=name, **kwargs)
-            if issubclass(data.dtype.type, np.integer):
+            if issubclass(data.dtype.type, np.signedinteger):
                 from .numeric import Int64Index
                 return Int64Index(data, copy=copy, dtype=dtype, name=name)
+            elif issubclass(data.dtype.type, np.unsignedinteger):
+                from .numeric import UInt64Index
+                return UInt64Index(data, copy=copy, dtype=dtype, name=name)
             elif issubclass(data.dtype.type, np.floating):
                 from .numeric import Float64Index
                 return Float64Index(data, copy=copy, dtype=dtype, name=name)
@@ -254,9 +268,13 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
             if dtype is None:
                 inferred = lib.infer_dtype(subarr)
                 if inferred == 'integer':
-                    from .numeric import Int64Index
-                    return Int64Index(subarr.astype('i8'), copy=copy,
-                                      name=name)
+                    from .numeric import Int64Index, UInt64Index
+                    try:
+                        return Int64Index(subarr.astype('i8'), copy=copy,
+                                          name=name)
+                    except OverflowError:
+                        return UInt64Index(subarr.astype('u8'), copy=copy,
+                                           name=name)
                 elif inferred in ['floating', 'mixed-integer-float']:
                     from .numeric import Float64Index
                     return Float64Index(subarr, copy=copy, name=name)