ENH: add map_infer function, use to speed up Series.map/apply, close PR #355

Author: wesm

RELEASE.rst

@@ -50,8 +50,8 @@ pandas 0.5.1
   - Add `orient` option to `Panel.from_dict` to ease creation of mixed-type
     Panels (GH #359)
   - Add `DataFrame.from_dict` with similar `orient` option
-  - Can pass list of tuples or list of lists to `DataFrame.from_records` for
-    fast conversion to DataFrame (GH #357)
+  - Can now pass list of tuples or list of lists to `DataFrame.from_records`
+    for fast conversion to DataFrame (GH #357)
 
 **Improvements to existing features**
 
@@ -72,6 +72,9 @@ pandas 0.5.1
   - Raise exception if dateutil 2.0 installed on Python 2.x runtime (GH #346)
   - Significant GroupBy performance enhancement with multiple keys with many
     "empty" combinations
+  - New Cython vectorized function `map_infer` speeds up `Series.apply` and
+    `Series.map` significantly when passed elementwise Python function,
+    motivated by PR #355
 
 **Bug fixes**
 
@@ -114,6 +117,7 @@ Thanks
 - Jev Kuznetsov
 - Dieter Vandenbussche
 - rsamson
+- Aman Thakral
 
 pandas 0.5.0
 ============

pandas/core/series.py

@@ -1331,8 +1331,8 @@ def map(self, arg):
             new_values = common.take_1d(np.asarray(arg), indexer)
             return Series(new_values, index=self.index, name=self.name)
         else:
-            return Series([arg(x) for x in self], index=self.index,
-                          name=self.name)
+            mapped = lib.map_infer(self.values, arg)
+            return Series(mapped, index=self.index, name=self.name)
 
     def apply(self, func):
         """
@@ -1353,8 +1353,8 @@ def apply(self, func):
                 result = Series(result, index=self.index, name=self.name)
             return result
         except Exception:
-            return Series([func(x) for x in self], index=self.index,
-                          name=self.name)
+            mapped = lib.map_infer(self.values, func)
+            return Series(mapped, index=self.index, name=self.name)
 
     def align(self, other, join='outer', copy=True):
         """

pandas/src/tseries.pyx

@@ -186,7 +186,7 @@ def ismember(ndarray arr, set values):
     it = <flatiter> PyArray_IterNew(arr)
     n = len(arr)
     result = np.empty(n, dtype=np.uint8)
-    for i from 0 <= i < n:
+    for i in range(n):
         val = PyArray_GETITEM(arr, PyArray_ITER_DATA(it))
         if val in values:
             result[i] = 1
@@ -196,6 +196,35 @@ def ismember(ndarray arr, set values):
 
     return result.view(np.bool_)
 
+def map_infer(ndarray arr, object f):
+    '''
+    Substitute for np.vectorize with pandas-friendly dtype inference
+
+    Parameters
+    ----------
+    arr : ndarray
+    f : function
+
+    Returns
+    -------
+    mapped : ndarray
+    '''
+    cdef:
+        Py_ssize_t i, n
+        flatiter it
+        ndarray[object] result
+        object val
+
+    it = <flatiter> PyArray_IterNew(arr)
+    n = len(arr)
+    result = np.empty(n, dtype=object)
+    for i in range(n):
+        val = PyArray_GETITEM(arr, PyArray_ITER_DATA(it))
+        result[i] = f(val)
+        PyArray_ITER_NEXT(it)
+
+    return maybe_convert_objects(result)
+
 #----------------------------------------------------------------------
 # datetime / io related