PERF/COMPAT: define platform int to np.intp

AFAIK this only affects 64 bit python on Windows.    `numpy` wants an
`np.intp` (i8 on Windows) as a indexer for `take`, but pandas defines
a "platform int" as a `np.int_` (i4 on Windows).  This hits
performance twice, because we often start with i8, cast to i4, then
numpy will cast back to i8 in its `take`.

Author: Chris <[email protected]>

Closes #13972 from chris-b1/platform-int and squashes the following commits:
Closes #3033

322b11a [Chris] lint fixup
fc80938 [Chris] adjust for 32bit
84f38b2 [Chris] adjust test for platform independence
3ced5d5 [Chris] PERF/COMPAT: define platform int to np.intp

Author: chris-b1

doc/source/whatsnew/v0.19.0.txt

@@ -778,6 +778,44 @@ Note that the limitation is applied to ``fill_value`` which default is ``np.nan`
 - Bug in single row slicing on multi-type ``SparseDataFrame``s, types were previously forced to float (:issue:`13917`)
 - Bug in sparse indexing using ``SparseArray`` with ``bool`` dtype may return incorrect result  (:issue:`13985`)
 
+.. _whatsnew_0190.indexer_dtype:
+
+Indexer dtype Changes
+^^^^^^^^^^^^^^^^^^^^^
+
+.. note::
+
+   This change only affects 64 bit python running on Windows, and only affects relatively advanced
+   indexing operations
+
+Methods such as ``Index.get_indexer`` that return an indexer array, coerce that array to a "platform int", so that it can be
+directly used in 3rd party library operations like ``numpy.take``.  Previously, a platform int was defined as ``np.int_``
+which corresponds to a C integer, but the correct type, and what is being used now, is ``np.intp``, which corresponds
+to the C integer size that can hold a pointer. (:issue:`3033`, :issue:`13972`)
+
+These types are the same on many platform, but for 64 bit python on Windows,
+``np.int_`` is 32 bits, and ``np.intp`` is 64 bits.  Changing this behavior improves performance for many
+operations on that platform.
+
+Previous behaviour:
+
+.. code-block:: ipython
+
+   In [1]: i = pd.Index(['a', 'b', 'c'])
+
+   In [2]: i.get_indexer(['b', 'b', 'c']).dtype
+   Out[2]: dtype('int32')
+
+New behaviour:
+
+.. code-block:: ipython
+
+   In [1]: i = pd.Index(['a', 'b', 'c'])
+
+   In [2]: i.get_indexer(['b', 'b', 'c']).dtype
+   Out[2]: dtype('int64')
+
+
 .. _whatsnew_0190.deprecations:
 
 Deprecations

pandas/core/algorithms.py

@@ -259,7 +259,7 @@ def sort_mixed(values):
     new_labels = reverse_indexer.take(labels, mode='wrap')
     np.putmask(new_labels, mask, na_sentinel)
 
-    return ordered, new_labels
+    return ordered, _ensure_platform_int(new_labels)
 
 
 def factorize(values, sort=False, order=None, na_sentinel=-1, size_hint=None):

pandas/hashtable.pyx

@@ -64,10 +64,10 @@ cdef class Factorizer:
         mask = (labels == na_sentinel)
         # sort on
         if sort:
-            if labels.dtype != np.int_:
-                labels = labels.astype(np.int_)
+            if labels.dtype != np.intp:
+                labels = labels.astype(np.intp)
             sorter = self.uniques.to_array().argsort()
-            reverse_indexer = np.empty(len(sorter), dtype=np.int_)
+            reverse_indexer = np.empty(len(sorter), dtype=np.intp)
             reverse_indexer.put(sorter, np.arange(len(sorter)))
             labels = reverse_indexer.take(labels, mode='clip')
             labels[mask] = na_sentinel
@@ -100,11 +100,11 @@ cdef class Int64Factorizer:
 
         # sort on
         if sort:
-            if labels.dtype != np.int_:
-                labels = labels.astype(np.int_)
+            if labels.dtype != np.intp:
+                labels = labels.astype(np.intp)
 
             sorter = self.uniques.to_array().argsort()
-            reverse_indexer = np.empty(len(sorter), dtype=np.int_)
+            reverse_indexer = np.empty(len(sorter), dtype=np.intp)
             reverse_indexer.put(sorter, np.arange(len(sorter)))
 
             labels = reverse_indexer.take(labels)

pandas/indexes/base.py

@@ -2820,7 +2820,7 @@ def _get_leaf_sorter(labels):
             new_levels[level] = new_level
 
             if keep_order:  # just drop missing values. o.w. keep order
-                left_indexer = np.arange(len(left))
+                left_indexer = np.arange(len(left), dtype=np.intp)
                 mask = new_lev_labels != -1
                 if not mask.all():
                     new_labels = [lab[mask] for lab in new_labels]
@@ -2863,6 +2863,10 @@ def _get_leaf_sorter(labels):
             left_indexer, right_indexer = right_indexer, left_indexer
 
         if return_indexers:
+            left_indexer = (None if left_indexer is None
+                            else _ensure_platform_int(left_indexer))
+            right_indexer = (None if right_indexer is None
+                             else _ensure_platform_int(right_indexer))
             return join_index, left_indexer, right_indexer
         else:
             return join_index
@@ -2906,6 +2910,8 @@ def _join_monotonic(self, other, how='left', return_indexers=False):
             join_index = self._wrap_joined_index(join_index, other)
 
         if return_indexers:
+            lidx = None if lidx is None else _ensure_platform_int(lidx)
+            ridx = None if ridx is None else _ensure_platform_int(ridx)
             return join_index, lidx, ridx
         else:
             return join_index

pandas/src/algos_common_helper.pxi

@@ -2848,16 +2848,18 @@ def put2d_int64_float64(ndarray[int64_t, ndim=2, cast=True] values,
 # ensure_dtype
 #----------------------------------------------------------------------
 
-cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.int_)).descr.type_num
+cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.intp)).descr.type_num
 
 cpdef ensure_platform_int(object arr):
+    # GH3033, GH1392
+    # platform int is the size of the int pointer, e.g. np.intp
     if util.is_array(arr):
         if (<ndarray> arr).descr.type_num == PLATFORM_INT:
             return arr
         else:
-            return arr.astype(np.int_)
+            return arr.astype(np.intp)
     else:
-        return np.array(arr, dtype=np.int_)
+        return np.array(arr, dtype=np.intp)
 
 cpdef ensure_object(object arr):
     if util.is_array(arr):

pandas/src/algos_common_helper.pxi.in

@@ -548,16 +548,18 @@ def put2d_{{name}}_{{dest_type}}(ndarray[{{c_type}}, ndim=2, cast=True] values,
 # ensure_dtype
 #----------------------------------------------------------------------
 
-cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.int_)).descr.type_num
+cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.intp)).descr.type_num
 
 cpdef ensure_platform_int(object arr):
+    # GH3033, GH1392
+    # platform int is the size of the int pointer, e.g. np.intp
     if util.is_array(arr):
         if (<ndarray> arr).descr.type_num == PLATFORM_INT:
             return arr
         else:
-            return arr.astype(np.int_)
+            return arr.astype(np.intp)
     else:
-        return np.array(arr, dtype=np.int_)
+        return np.array(arr, dtype=np.intp)
 
 cpdef ensure_object(object arr):
     if util.is_array(arr):
@@ -600,4 +602,4 @@ cpdef ensure_{{name}}(object arr):
     else:
         return np.array(arr, dtype=np.{{dtype}})
 
-{{endfor}}
+{{endfor}}

pandas/src/join.pyx

@@ -32,7 +32,8 @@ float64 = np.dtype(np.float64)
 cdef double NaN = <double> np.NaN
 cdef double nan = NaN
 
-from pandas.algos import groupsort_indexer
+from pandas.algos import groupsort_indexer, ensure_platform_int
+from pandas.core.algorithms import take_nd
 
 include "joins_func_helper.pxi"
 
@@ -148,16 +149,14 @@ def left_outer_join(ndarray[int64_t] left, ndarray[int64_t] right,
             # no multiple matches for any row on the left
             # this is a short-cut to avoid groupsort_indexer
             # otherwise, the `else` path also works in this case
-            if left_sorter.dtype != np.int_:
-                left_sorter = left_sorter.astype(np.int_)
+            left_sorter = ensure_platform_int(left_sorter)
 
-            rev = np.empty(len(left), dtype=np.int_)
+            rev = np.empty(len(left), dtype=np.intp)
             rev.put(left_sorter, np.arange(len(left)))
         else:
             rev, _ = groupsort_indexer(left_indexer, len(left))
 
-        if rev.dtype != np.int_:
-            rev = rev.astype(np.int_)
+        rev = ensure_platform_int(rev)
         right_indexer = right_indexer.take(rev)
         left_indexer = left_indexer.take(rev)
 
@@ -228,11 +227,8 @@ def full_outer_join(ndarray[int64_t] left, ndarray[int64_t] right,
 
 
 def _get_result_indexer(sorter, indexer):
-    if indexer.dtype != np.int_:
-        indexer = indexer.astype(np.int_)
     if len(sorter) > 0:
-        res = sorter.take(indexer)
-        np.putmask(res, indexer == -1, -1)
+        res = take_nd(sorter, indexer, fill_value=-1)
     else:
         # length-0 case
         res = np.empty(len(indexer), dtype=np.int64)

pandas/tests/frame/test_operators.py

@@ -1204,7 +1204,7 @@ def test_alignment_non_pandas(self):
 
         align = pd.core.ops._align_method_FRAME
 
-        for val in [[1, 2, 3], (1, 2, 3), np.array([1, 2, 3], dtype=np.intp)]:
+        for val in [[1, 2, 3], (1, 2, 3), np.array([1, 2, 3], dtype=np.int64)]:
 
             tm.assert_series_equal(align(df, val, 'index'),
                                    Series([1, 2, 3], index=df.index))

pandas/tests/indexes/common.py

@@ -110,7 +110,7 @@ def f():
 
     def test_reindex_base(self):
         idx = self.create_index()
-        expected = np.arange(idx.size)
+        expected = np.arange(idx.size, dtype=np.intp)
 
         actual = idx.get_indexer(idx)
         tm.assert_numpy_array_equal(expected, actual)

pandas/tests/indexes/test_base.py

@@ -975,10 +975,10 @@ def test_get_indexer(self):
         idx2 = Index([2, 4, 6])
 
         r1 = idx1.get_indexer(idx2)
-        assert_almost_equal(r1, np.array([1, 3, -1]))
+        assert_almost_equal(r1, np.array([1, 3, -1], dtype=np.intp))
 
         r1 = idx2.get_indexer(idx1, method='pad')
-        e1 = np.array([-1, 0, 0, 1, 1])
+        e1 = np.array([-1, 0, 0, 1, 1], dtype=np.intp)
         assert_almost_equal(r1, e1)
 
         r2 = idx2.get_indexer(idx1[::-1], method='pad')
@@ -988,7 +988,7 @@ def test_get_indexer(self):
         assert_almost_equal(r1, rffill1)
 
         r1 = idx2.get_indexer(idx1, method='backfill')
-        e1 = np.array([0, 0, 1, 1, 2])
+        e1 = np.array([0, 0, 1, 1, 2], dtype=np.intp)
         assert_almost_equal(r1, e1)
 
         rbfill1 = idx2.get_indexer(idx1, method='bfill')
@@ -1013,25 +1013,30 @@ def test_get_indexer_nearest(self):
         all_methods = ['pad', 'backfill', 'nearest']
         for method in all_methods:
             actual = idx.get_indexer([0, 5, 9], method=method)
-            tm.assert_numpy_array_equal(actual, np.array([0, 5, 9]))
+            tm.assert_numpy_array_equal(actual, np.array([0, 5, 9],
+                                                         dtype=np.intp))
 
             actual = idx.get_indexer([0, 5, 9], method=method, tolerance=0)
-            tm.assert_numpy_array_equal(actual, np.array([0, 5, 9]))
+            tm.assert_numpy_array_equal(actual, np.array([0, 5, 9],
+                                                         dtype=np.intp))
 
         for method, expected in zip(all_methods, [[0, 1, 8], [1, 2, 9],
                                                   [0, 2, 9]]):
             actual = idx.get_indexer([0.2, 1.8, 8.5], method=method)
-            tm.assert_numpy_array_equal(actual, np.array(expected))
+            tm.assert_numpy_array_equal(actual, np.array(expected,
+                                                         dtype=np.intp))
 
             actual = idx.get_indexer([0.2, 1.8, 8.5], method=method,
                                      tolerance=1)
-            tm.assert_numpy_array_equal(actual, np.array(expected))
+            tm.assert_numpy_array_equal(actual, np.array(expected,
+                                                         dtype=np.intp))
 
         for method, expected in zip(all_methods, [[0, -1, -1], [-1, 2, -1],
                                                   [0, 2, -1]]):
             actual = idx.get_indexer([0.2, 1.8, 8.5], method=method,
                                      tolerance=0.2)
-            tm.assert_numpy_array_equal(actual, np.array(expected))
+            tm.assert_numpy_array_equal(actual, np.array(expected,
+                                                         dtype=np.intp))
 
         with tm.assertRaisesRegexp(ValueError, 'limit argument'):
             idx.get_indexer([1, 0], method='nearest', limit=1)
@@ -1042,22 +1047,24 @@ def test_get_indexer_nearest_decreasing(self):
         all_methods = ['pad', 'backfill', 'nearest']
         for method in all_methods:
             actual = idx.get_indexer([0, 5, 9], method=method)
-            tm.assert_numpy_array_equal(actual, np.array([9, 4, 0]))
+            tm.assert_numpy_array_equal(actual, np.array([9, 4, 0],
+                                                         dtype=np.intp))
 
         for method, expected in zip(all_methods, [[8, 7, 0], [9, 8, 1],
                                                   [9, 7, 0]]):
             actual = idx.get_indexer([0.2, 1.8, 8.5], method=method)
-            tm.assert_numpy_array_equal(actual, np.array(expected))
+            tm.assert_numpy_array_equal(actual, np.array(expected,
+                                                         dtype=np.intp))
 
     def test_get_indexer_strings(self):
         idx = pd.Index(['b', 'c'])
 
         actual = idx.get_indexer(['a', 'b', 'c', 'd'], method='pad')
-        expected = np.array([-1, 0, 1, 1])
+        expected = np.array([-1, 0, 1, 1], dtype=np.intp)
         tm.assert_numpy_array_equal(actual, expected)
 
         actual = idx.get_indexer(['a', 'b', 'c', 'd'], method='backfill')
-        expected = np.array([0, 0, 1, -1])
+        expected = np.array([0, 0, 1, -1], dtype=np.intp)
         tm.assert_numpy_array_equal(actual, expected)
 
         with tm.assertRaises(TypeError):

pandas/tests/indexes/test_category.py

@@ -336,7 +336,7 @@ def test_reindex_base(self):
 
         # determined by cat ordering
         idx = self.create_index()
-        expected = np.array([4, 0, 1, 5, 2, 3])
+        expected = np.array([4, 0, 1, 5, 2, 3], dtype=np.intp)
 
         actual = idx.get_indexer(idx)
         tm.assert_numpy_array_equal(expected, actual)
@@ -403,7 +403,7 @@ def test_get_indexer(self):
 
         for indexer in [idx2, list('abf'), Index(list('abf'))]:
             r1 = idx1.get_indexer(idx2)
-            assert_almost_equal(r1, np.array([0, 1, 2, -1]))
+            assert_almost_equal(r1, np.array([0, 1, 2, -1], dtype=np.intp))
 
         self.assertRaises(NotImplementedError,
                           lambda: idx2.get_indexer(idx1, method='pad'))