ENH: Create and propagate UInt64Index

Author: gfyoung

doc/source/whatsnew/v0.20.0.txt

@@ -88,6 +88,24 @@ support for bz2 compression in the python 2 c-engine improved (:issue:`14874`).
    df = pd.read_table(url, compression='bz2')  # explicitly specify compression
    df.head(2)
 
+.. _whatsnew_0200.enhancements.uint64_support:
+
+Pandas has significantly improved support for operations involving unsigned,
+or purely non-negative, integers. Previously, handling these integers would
+result in improper rounding or data-type casting, leading to incorrect results.
+Notably, a new numerical index, `UInt64Index`, has been created (:issue:`14937`)
+
+.. ipython:: python
+
+   idx = pd.UInt64Index([1, 2, 3])
+   df = pd.DataFrame(['a', 'b', 'c'], index=idx)
+   df.index
+
+- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`)
+- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
+- Bug in ``DataFrame`` construction in which unsigned 64-bit integer elements were being converted to objects (:issue:`14881`)
+- Bug in ``pd.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14915`)
+
 .. _whatsnew_0200.enhancements.other:
 
 Other enhancements
@@ -279,7 +297,6 @@ Bug Fixes
 ~~~~~~~~~
 
 - Bug in ``TimedeltaIndex`` addition where overflow was being allowed without error (:issue:`14816`)
-- Bug in ``DataFrame`` construction in which unsigned 64-bit integer elements were being converted to objects (:issue:`14881`)
 - Bug in ``astype()`` where ``inf`` values were incorrectly converted to integers. Now raises error now with ``astype()`` for Series and DataFrames (:issue:`14265`)
 - Bug in ``DataFrame(..).apply(to_numeric)`` when values are of type decimal.Decimal. (:issue:`14827`)
 - Bug in ``describe()`` when passing a numpy array which does not contain the median to the ``percentiles`` keyword argument (:issue:`14908`)
@@ -299,8 +316,6 @@ Bug Fixes
 
 
 
-- Bug in ``Series.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14721`)
-- Bug in ``pd.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14915`)
 
 
 
@@ -316,5 +331,4 @@ Bug Fixes
 
 
 - Require at least 0.23 version of cython to avoid problems with character encodings (:issue:`14699`)
-- Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`)
 - Bug in ``pd.pivot_table()`` where no error was raised when values argument was not in the columns (:issue:`14938`)

pandas/api/tests/test_api.py

@@ -53,7 +53,7 @@ class TestPDApi(Base, tm.TestCase):
     classes = ['Categorical', 'CategoricalIndex', 'DataFrame', 'DateOffset',
                'DatetimeIndex', 'ExcelFile', 'ExcelWriter', 'Float64Index',
                'Grouper', 'HDFStore', 'Index', 'Int64Index', 'MultiIndex',
-               'Period', 'PeriodIndex', 'RangeIndex',
+               'Period', 'PeriodIndex', 'RangeIndex', 'UInt64Index',
                'Series', 'SparseArray', 'SparseDataFrame',
                'SparseSeries', 'TimeGrouper', 'Timedelta',
                'TimedeltaIndex', 'Timestamp']

pandas/core/api.py

@@ -10,7 +10,8 @@
 from pandas.core.groupby import Grouper
 from pandas.formats.format import set_eng_float_format
 from pandas.core.index import (Index, CategoricalIndex, Int64Index,
-                               RangeIndex, Float64Index, MultiIndex)
+                               UInt64Index, RangeIndex, Float64Index,
+                               MultiIndex)
 
 from pandas.core.series import Series, TimeSeries
 from pandas.core.frame import DataFrame

pandas/core/indexing.py

@@ -3,7 +3,8 @@
 import numpy as np
 from pandas.compat import range, zip
 import pandas.compat as compat
-from pandas.types.generic import ABCDataFrame, ABCPanel, ABCSeries
+from pandas.types.generic import (ABCDataFrame, ABCPanel,
+                                  ABCSeries, ABCUInt64Index)
 from pandas.types.common import (is_integer_dtype,
                                  is_integer, is_float,
                                  is_categorical_dtype,
@@ -859,15 +860,22 @@ def _convert_for_reindex(self, key, axis=0):
             return labels[key]
         else:
             if isinstance(key, Index):
-                # want Index objects to pass through untouched
-                keyarr = key
+                if isinstance(labels, ABCUInt64Index) and key.is_integer():
+                    keyarr = key.astype(np.uint64)
+                else:
+                    keyarr = key
             else:
                 # asarray can be unsafe, NumPy strings are weird
                 keyarr = _asarray_tuplesafe(key)
 
-            if is_integer_dtype(keyarr) and not labels.is_integer():
-                keyarr = _ensure_platform_int(keyarr)
-                return labels.take(keyarr)
+            if is_integer_dtype(keyarr):
+                if isinstance(labels, ABCUInt64Index):
+                    keyarr = _asarray_tuplesafe(keyarr, dtype=np.uint64)
+                    return keyarr
+
+                if not labels.is_integer():
+                    keyarr = _ensure_platform_int(keyarr)
+                    return labels.take(keyarr)
 
             return keyarr
 
@@ -1043,12 +1051,17 @@ def _getitem_iterable(self, key, axis=0):
             return self.obj.take(inds, axis=axis, convert=False)
         else:
             if isinstance(key, Index):
-                # want Index objects to pass through untouched
-                keyarr = key
+                if isinstance(labels, ABCUInt64Index) and key.is_integer():
+                    keyarr = key.astype(np.uint64)
+                else:
+                    keyarr = key
             else:
-                # asarray can be unsafe, NumPy strings are weird
                 keyarr = _asarray_tuplesafe(key)
 
+                if (is_integer_dtype(keyarr) and
+                        isinstance(labels, ABCUInt64Index)):
+                    keyarr = _asarray_tuplesafe(key, dtype=np.uint64)
+
             if is_categorical_dtype(labels):
                 keyarr = labels._shallow_copy(keyarr)
 

pandas/index.pyx

@@ -363,115 +363,6 @@ cdef class IndexEngine:
 
         return result[0:count], missing[0:count_missing]
 
-cdef class Int64Engine(IndexEngine):
-
-    cdef _get_index_values(self):
-        return algos.ensure_int64(self.vgetter())
-
-    cdef _make_hash_table(self, n):
-        return _hash.Int64HashTable(n)
-
-    def _call_monotonic(self, values):
-        return algos.is_monotonic_int64(values, timelike=False)
-
-    def get_pad_indexer(self, other, limit=None):
-        return algos.pad_int64(self._get_index_values(), other,
-                               limit=limit)
-
-    def get_backfill_indexer(self, other, limit=None):
-        return algos.backfill_int64(self._get_index_values(), other,
-                                    limit=limit)
-
-    cdef _check_type(self, object val):
-        hash(val)
-        if util.is_bool_object(val):
-            raise KeyError(val)
-        elif util.is_float_object(val):
-            raise KeyError(val)
-
-    cdef _maybe_get_bool_indexer(self, object val):
-        cdef:
-            ndarray[uint8_t, cast=True] indexer
-            ndarray[int64_t] values
-            int count = 0
-            Py_ssize_t i, n
-            int64_t ival
-            int last_true
-
-        if not util.is_integer_object(val):
-            raise KeyError(val)
-
-        ival = val
-
-        values = self._get_index_values()
-        n = len(values)
-
-        result = np.empty(n, dtype=bool)
-        indexer = result.view(np.uint8)
-
-        for i in range(n):
-            if values[i] == val:
-                count += 1
-                indexer[i] = 1
-                last_true = i
-            else:
-                indexer[i] = 0
-
-        if count == 0:
-            raise KeyError(val)
-        if count == 1:
-            return last_true
-
-        return result
-
-cdef class Float64Engine(IndexEngine):
-
-    cdef _make_hash_table(self, n):
-        return _hash.Float64HashTable(n)
-
-    cdef _get_index_values(self):
-        return algos.ensure_float64(self.vgetter())
-
-    cdef _maybe_get_bool_indexer(self, object val):
-        cdef:
-            ndarray[uint8_t] indexer
-            ndarray[float64_t] values
-            int count = 0
-            Py_ssize_t i, n
-            int last_true
-
-        values = self._get_index_values()
-        n = len(values)
-
-        result = np.empty(n, dtype=bool)
-        indexer = result.view(np.uint8)
-
-        for i in range(n):
-            if values[i] == val:
-                count += 1
-                indexer[i] = 1
-                last_true = i
-            else:
-                indexer[i] = 0
-
-        if count == 0:
-            raise KeyError(val)
-        if count == 1:
-            return last_true
-
-        return result
-
-    def _call_monotonic(self, values):
-        return algos.is_monotonic_float64(values, timelike=False)
-
-    def get_pad_indexer(self, other, limit=None):
-        return algos.pad_float64(self._get_index_values(), other,
-                                    limit=limit)
-
-    def get_backfill_indexer(self, other, limit=None):
-        return algos.backfill_float64(self._get_index_values(), other,
-                                         limit=limit)
-
 
 cdef Py_ssize_t _bin_search(ndarray values, object val) except -1:
     cdef:
@@ -510,22 +401,6 @@ _backfill_functions = {
     'float64': algos.backfill_float64
 }
 
-cdef class ObjectEngine(IndexEngine):
-
-    cdef _make_hash_table(self, n):
-        return _hash.PyObjectHashTable(n)
-
-    def _call_monotonic(self, values):
-        return algos.is_monotonic_object(values, timelike=False)
-
-    def get_pad_indexer(self, other, limit=None):
-        return algos.pad_object(self._get_index_values(), other,
-                                   limit=limit)
-
-    def get_backfill_indexer(self, other, limit=None):
-        return algos.backfill_object(self._get_index_values(), other,
-                                        limit=limit)
-
 
 cdef class DatetimeEngine(Int64Engine):
 
@@ -668,3 +543,7 @@ cdef inline _to_i8(object val):
 
 cdef inline bint _is_utc(object tz):
     return tz is UTC or isinstance(tz, _du_utc)
+
+
+# Generated from template.
+include "index_class_helper.pxi"

pandas/indexes/api.py

@@ -4,7 +4,7 @@
 from pandas.indexes.category import CategoricalIndex  # noqa
 from pandas.indexes.multi import MultiIndex  # noqa
 from pandas.indexes.numeric import (NumericIndex, Float64Index,  # noqa
-                                    Int64Index)
+                                    Int64Index, UInt64Index)
 from pandas.indexes.range import RangeIndex  # noqa
 
 import pandas.core.common as com
@@ -13,7 +13,7 @@
 # TODO: there are many places that rely on these private methods existing in
 # pandas.core.index
 __all__ = ['Index', 'MultiIndex', 'NumericIndex', 'Float64Index', 'Int64Index',
-           'CategoricalIndex', 'RangeIndex',
+           'CategoricalIndex', 'RangeIndex', 'UInt64Index',
            'InvalidIndexError',
            '_new_Index',
            '_ensure_index', '_get_na_value', '_get_combined_index',

pandas/indexes/base.py

@@ -199,14 +199,25 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                             data = np.array(data, copy=copy, dtype=dtype)
                         elif inferred in ['floating', 'mixed-integer-float']:
 
-                            # if we are actually all equal to integers
+                            # If we are actually all equal to integers,
                             # then coerce to integer
-                            from .numeric import Int64Index, Float64Index
+                            from .numeric import (Int64Index, UInt64Index,
+                                                  Float64Index)
                             try:
                                 res = data.astype('i8')
                                 if (res == data).all():
                                     return Int64Index(res, copy=copy,
                                                       name=name)
+                            except (OverflowError, TypeError, ValueError):
+                                pass
+
+                            # Conversion to int64 failed (possibly due to
+                            # overflow), so let's try now with uint64.
+                            try:
+                                res = data.astype('u8')
+                                if (res == data).all():
+                                    return UInt64Index(res, copy=copy,
+                                                       name=name)
                             except (TypeError, ValueError):
                                 pass
 
@@ -235,9 +246,12 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
                                                IncompatibleFrequency)
             if isinstance(data, PeriodIndex):
                 return PeriodIndex(data, copy=copy, name=name, **kwargs)
-            if issubclass(data.dtype.type, np.integer):
+            if issubclass(data.dtype.type, np.signedinteger):
                 from .numeric import Int64Index
                 return Int64Index(data, copy=copy, dtype=dtype, name=name)
+            elif issubclass(data.dtype.type, np.unsignedinteger):
+                from .numeric import UInt64Index
+                return UInt64Index(data, copy=copy, dtype=dtype, name=name)
             elif issubclass(data.dtype.type, np.floating):
                 from .numeric import Float64Index
                 return Float64Index(data, copy=copy, dtype=dtype, name=name)
@@ -254,9 +268,13 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None,
             if dtype is None:
                 inferred = lib.infer_dtype(subarr)
                 if inferred == 'integer':
-                    from .numeric import Int64Index
-                    return Int64Index(subarr.astype('i8'), copy=copy,
-                                      name=name)
+                    from .numeric import Int64Index, UInt64Index
+                    try:
+                        return Int64Index(subarr.astype('i8'), copy=copy,
+                                          name=name)
+                    except OverflowError:
+                        return UInt64Index(subarr.astype('u8'), copy=copy,
+                                           name=name)
                 elif inferred in ['floating', 'mixed-integer-float']:
                     from .numeric import Float64Index
                     return Float64Index(subarr, copy=copy, name=name)