PERF/COMPAT: define platform int to np.intp

Author: chris-b1

doc/source/whatsnew/v0.19.0.txt

@@ -767,6 +767,40 @@ Note that the limitation is applied to ``fill_value`` which default is ``np.nan`
 - Bug in ``SparseSeries.abs`` incorrectly keeps negative ``fill_value`` (:issue:`13853`)
 - Bug in single row slicing on multi-type ``SparseDataFrame``s, types were previously forced to float (:issue:`13917`)
 
+Indexer dtype Changes
+^^^^^^^^^^^^^^^^^^^^^
+
+.. note::
+
+   This change only affects 64 bit python running on Windows, and only affects relatively advanced
+   indexing operations
+
+Methods such as ``Index.get_indexer`` that return an indexer array coerce that array to a "platform int", so that it can be
+directly used in 3rd party library operations like ``numpy.take``.  Previously, a platform int was defined as ``np.int_``
+which corresponds to a C integer - but the correct type, and what is being used now, is ``np.intp``, which corresponds
+to the C integer size that can hold a pointer.
+
+These types are the same on many platform, but for 64 bit python on Windows,
+``np.int_`` is 32 bits, and ``np.intp`` is 64 bits.  Changing this behavior improves performance for many
+operations on that platform.
+
+Previous behaviour:
+
+.. code-block:: ipython
+
+   In [1]: i = pd.Index(['a', 'b', 'c'])
+
+   In [2]: i.get_indexer(['b', 'b', 'c']).dtype
+   Out[2]: dtype('int32')
+
+New behaviour:
+
+.. ipython :: python
+
+   i = pd.Index(['a', 'b', 'c'])
+   i.get_indexer(['b', 'b', 'c']).dtype
+
+
 .. _whatsnew_0190.deprecations:
 
 Deprecations

pandas/src/algos_common_helper.pxi

@@ -2848,16 +2848,16 @@ def put2d_int64_float64(ndarray[int64_t, ndim=2, cast=True] values,
 # ensure_dtype
 #----------------------------------------------------------------------
 
-cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.int_)).descr.type_num
+cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.intp)).descr.type_num
 
 cpdef ensure_platform_int(object arr):
     if util.is_array(arr):
         if (<ndarray> arr).descr.type_num == PLATFORM_INT:
             return arr
         else:
-            return arr.astype(np.int_)
+            return arr.astype(np.intp)
     else:
-        return np.array(arr, dtype=np.int_)
+        return np.array(arr, dtype=np.intp)
 
 cpdef ensure_object(object arr):
     if util.is_array(arr):

pandas/src/algos_common_helper.pxi.in

@@ -548,16 +548,16 @@ def put2d_{{name}}_{{dest_type}}(ndarray[{{c_type}}, ndim=2, cast=True] values,
 # ensure_dtype
 #----------------------------------------------------------------------
 
-cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.int_)).descr.type_num
+cdef int PLATFORM_INT = (<ndarray> np.arange(0, dtype=np.intp)).descr.type_num
 
 cpdef ensure_platform_int(object arr):
     if util.is_array(arr):
         if (<ndarray> arr).descr.type_num == PLATFORM_INT:
             return arr
         else:
-            return arr.astype(np.int_)
+            return arr.astype(np.intp)
     else:
-        return np.array(arr, dtype=np.int_)
+        return np.array(arr, dtype=np.intp)
 
 cpdef ensure_object(object arr):
     if util.is_array(arr):
@@ -600,4 +600,4 @@ cpdef ensure_{{name}}(object arr):
     else:
         return np.array(arr, dtype=np.{{dtype}})
 
-{{endfor}}
+{{endfor}}

pandas/src/join.pyx

@@ -32,7 +32,8 @@ float64 = np.dtype(np.float64)
 cdef double NaN = <double> np.NaN
 cdef double nan = NaN
 
-from pandas.algos import groupsort_indexer
+from pandas.algos import groupsort_indexer, ensure_platform_int
+from pandas.core.algorithms import take_nd
 
 include "joins_func_helper.pxi"
 
@@ -148,16 +149,14 @@ def left_outer_join(ndarray[int64_t] left, ndarray[int64_t] right,
             # no multiple matches for any row on the left
             # this is a short-cut to avoid groupsort_indexer
             # otherwise, the `else` path also works in this case
-            if left_sorter.dtype != np.int_:
-                left_sorter = left_sorter.astype(np.int_)
+            left_sorter = ensure_platform_int(left_sorter)
 
-            rev = np.empty(len(left), dtype=np.int_)
+            rev = np.empty(len(left), dtype=np.intp)
             rev.put(left_sorter, np.arange(len(left)))
         else:
             rev, _ = groupsort_indexer(left_indexer, len(left))
 
-        if rev.dtype != np.int_:
-            rev = rev.astype(np.int_)
+        rev = ensure_platform_int(rev)
         right_indexer = right_indexer.take(rev)
         left_indexer = left_indexer.take(rev)
 
@@ -228,11 +227,8 @@ def full_outer_join(ndarray[int64_t] left, ndarray[int64_t] right,
 
 
 def _get_result_indexer(sorter, indexer):
-    if indexer.dtype != np.int_:
-        indexer = indexer.astype(np.int_)
     if len(sorter) > 0:
-        res = sorter.take(indexer)
-        np.putmask(res, indexer == -1, -1)
+        res = take_nd(sorter, indexer, fill_value=-1)
     else:
         # length-0 case
         res = np.empty(len(indexer), dtype=np.int64)

pandas/tools/merge.py

@@ -572,7 +572,7 @@ def _maybe_add_join_keys(self, result, left_indexer, right_indexer):
                     if name in self.left:
 
                         if left_has_missing is None:
-                            left_has_missing = any(left_indexer == -1)
+                            left_has_missing = (left_indexer == -1).any()
 
                         if left_has_missing:
                             take_right = self.right_join_keys[i]
@@ -584,7 +584,7 @@ def _maybe_add_join_keys(self, result, left_indexer, right_indexer):
                     elif name in self.right:
 
                         if right_has_missing is None:
-                            right_has_missing = any(right_indexer == -1)
+                            right_has_missing = (right_indexer == -1).any()
 
                         if right_has_missing:
                             take_left = self.left_join_keys[i]