ENH: optimize Cython take functions with memmove where possible

Author: stephenwlin

pandas/core/common.py

@@ -697,8 +697,6 @@ def _maybe_promote(dtype, fill_value=np.nan):
                 # object (but numpy 1.6.1 doesn't do this properly)
                 fill_value = tslib.iNaT 
     elif is_float(fill_value):
-        if fill_value is None:
-            fill_value = np.nan
         if issubclass(dtype.type, np.bool_):
             dtype = np.object_
         elif issubclass(dtype.type, np.integer):

pandas/src/generate_code.py

@@ -5,6 +5,8 @@
 cimport numpy as np
 cimport cython
 
+from libc.string cimport memmove
+
 from numpy cimport *
 
 from cpython cimport (PyDict_New, PyDict_GetItem, PyDict_SetItem,
@@ -86,6 +88,23 @@ def take_2d_axis0_%(name)s_%(dest)s(ndarray[%(c_type_in)s, ndim=2] values,
     k = values.shape[1]
 
     fv = fill_value
+
+    IF %(can_copy)s:
+        cdef:
+            %(c_type_out)s *v, *o
+
+        if values.flags.c_contiguous and out.flags.c_contiguous:
+            for i from 0 <= i < n:
+                idx = indexer[i]
+                if idx == -1:
+                    for j from 0 <= j < k:
+                        outbuf[i, j] = fv
+                else:
+                    v = &values[idx, 0]
+                    o = &outbuf[i, 0]
+                    memmove(o, v, <size_t>(sizeof(%(c_type_out)s) * k))
+            return
+
     for i from 0 <= i < n:
         idx = indexer[i]
         if idx == -1:
@@ -109,8 +128,25 @@ def take_2d_axis1_%(name)s_%(dest)s(ndarray[%(c_type_in)s, ndim=2] values,
 
     n = len(values)
     k = len(indexer)
-
+    
     fv = fill_value
+
+    IF %(can_copy)s:
+        cdef:
+            %(c_type_out)s *v, *o
+
+        if values.flags.f_contiguous and out.flags.f_contiguous:
+            for j from 0 <= j < k:
+                idx = indexer[j]
+                if idx == -1:
+                    for i from 0 <= i < n:
+                        outbuf[i, j] = fv
+                else:
+                    v = &values[0, idx]
+                    o = &outbuf[0, j]
+                    memmove(o, v, <size_t>(sizeof(%(c_type_out)s) * n))
+            return
+
     for j from 0 <= j < k:
         idx = indexer[j]
         if idx == -1:
@@ -2115,39 +2151,40 @@ def generate_put_template(template, use_ints = True, use_floats = True):
     return output.getvalue()
 
 def generate_take_template(template, exclude=None):
-    # name, dest, ctypein, ctypeout, preval, postval
+    # name, dest, ctypein, ctypeout, preval, postval, cancopy
     function_list = [
-        ('bool', 'bool', 'uint8_t', 'uint8_t', '', ''),
+        ('bool', 'bool', 'uint8_t', 'uint8_t', '', '', True),
         ('bool', 'object', 'uint8_t', 'object',
-         'True if ', ' > 0 else False'),
-        ('int8', 'int8', 'int8_t', 'int8_t', '', ''),
-        ('int8', 'int32', 'int8_t', 'int32_t', '', ''),
-        ('int8', 'int64', 'int8_t', 'int64_t', '', ''),
-        ('int8', 'float64', 'int8_t', 'float64_t', '', ''),
-        ('int16', 'int16', 'int16_t', 'int16_t', '', ''),
-        ('int16', 'int32', 'int16_t', 'int32_t', '', ''),
-        ('int16', 'int64', 'int16_t', 'int64_t', '', ''),
-        ('int16', 'float64', 'int16_t', 'float64_t', '', ''),
-        ('int32', 'int32', 'int32_t', 'int32_t', '', ''),
-        ('int32', 'int64', 'int32_t', 'int64_t', '', ''),
-        ('int32', 'float64', 'int32_t', 'float64_t', '', ''),
-        ('int64', 'int64', 'int64_t', 'int64_t', '', ''),
-        ('int64', 'float64', 'int64_t', 'float64_t', '', ''),
-        ('float32', 'float32', 'float32_t', 'float32_t', '', ''),
-        ('float32', 'float64', 'float32_t', 'float64_t', '', ''),
-        ('float64', 'float64', 'float64_t', 'float64_t', '', ''),
-        ('object', 'object', 'object', 'object', '', '')
+         'True if ', ' > 0 else False', False),
+        ('int8', 'int8', 'int8_t', 'int8_t', '', '', True),
+        ('int8', 'int32', 'int8_t', 'int32_t', '', '', False),
+        ('int8', 'int64', 'int8_t', 'int64_t', '', '', False),
+        ('int8', 'float64', 'int8_t', 'float64_t', '', '', False),
+        ('int16', 'int16', 'int16_t', 'int16_t', '', '', True),
+        ('int16', 'int32', 'int16_t', 'int32_t', '', '', False),
+        ('int16', 'int64', 'int16_t', 'int64_t', '', '', False),
+        ('int16', 'float64', 'int16_t', 'float64_t', '', '', False),
+        ('int32', 'int32', 'int32_t', 'int32_t', '', '', True),
+        ('int32', 'int64', 'int32_t', 'int64_t', '', '', False),
+        ('int32', 'float64', 'int32_t', 'float64_t', '', '', False),
+        ('int64', 'int64', 'int64_t', 'int64_t', '', '', True),
+        ('int64', 'float64', 'int64_t', 'float64_t', '', '', False),
+        ('float32', 'float32', 'float32_t', 'float32_t', '', '', True),
+        ('float32', 'float64', 'float32_t', 'float64_t', '', '', False),
+        ('float64', 'float64', 'float64_t', 'float64_t', '', '', True),
+        ('object', 'object', 'object', 'object', '', '', False)
     ]
 
     output = StringIO()
     for (name, dest, c_type_in, c_type_out, 
-         preval, postval) in function_list:
+         preval, postval, can_copy) in function_list:
         if exclude is not None and name in exclude:
             continue
 
         func = template % {'name': name, 'dest': dest,
                            'c_type_in': c_type_in, 'c_type_out': c_type_out,
-                           'preval': preval, 'postval': postval}
+                           'preval': preval, 'postval': postval,
+                           'can_copy': 'True' if can_copy else 'False'}
         output.write(func)
     return output.getvalue()