ENH: be more careful about performance in list conversion when passed to _ensure_index. per #2336

Author: wesm

pandas/core/frame.py

@@ -2249,7 +2249,8 @@ def lookup(self, row_labels, col_labels):
 
         n = len(row_labels)
         if n != len(col_labels):
-            raise AssertionError('Row labels must have same size as col labels')
+            raise AssertionError('Row labels must have same size as '
+                                 'column labels')
 
         thresh = 1000
         if not self._is_mixed_type or n > thresh:

pandas/core/index.py

@@ -2527,15 +2527,13 @@ def _ensure_index(index_like):
         return Index(index_like, name=index_like.name)
 
     if isinstance(index_like, list):
-        klasses = (list, np.ndarray)
-        all_arrays = all(isinstance(x, klasses) for x in index_like)
+        # #2200 ?
+        converted, all_arrays = lib.clean_index_list(index_like)
 
-        if len(index_like) > 0 and all_arrays:
-            return MultiIndex.from_arrays(index_like)
+        if len(converted) > 0 and all_arrays:
+            return MultiIndex.from_arrays(converted)
         else:
-            # #2200 ?
-            index_like = [tuple(x) if isinstance(x, klasses) else x
-                          for x in index_like]
+            index_like = converted
 
     return Index(index_like)
 

pandas/src/tseries.pyx

@@ -707,6 +707,36 @@ def astype_intsafe(ndarray[object] arr, new_dtype):
 
     return result
 
+def clean_index_list(list obj):
+    '''
+    Utility used in pandas.core.index._ensure_index
+    '''
+    cdef:
+        ndarray[object] converted
+        Py_ssize_t i, n = len(obj)
+        object v
+        bint all_arrays = 1
+
+    for i in range(n):
+        v = obj[i]
+        if not (PyList_Check(v) or cnp.PyArray_Check(v)):
+            all_arrays = 0
+            break
+
+    if all_arrays:
+        return obj, all_arrays
+
+    converted = np.empty(n, dtype=object)
+    for i in range(n):
+        v = obj[i]
+        if PyList_Check(v) or cnp.PyArray_Check(v):
+            converted[i] = tuple(v)
+        else:
+            converted[i] = v
+
+    return maybe_convert_objects(converted), 0
+
+
 include "hashtable.pyx"
 include "datetime.pyx"
 include "skiplist.pyx"

vb_suite/frame_methods.py

@@ -13,8 +13,8 @@
 
 row_labels = list(df.index[::10])[:900]
 col_labels = list(df.columns) * 100
-row_labels_all = list(df.index) * len(df.columns)
-col_labels_all = list(df.columns) * len(df.index)
+row_labels_all = np.array(list(df.index) * len(df.columns), dtype='object')
+col_labels_all = np.array(list(df.columns) * len(df.index), dtype='object')
 """
 
 frame_fancy_lookup = Benchmark('df.lookup(row_labels, col_labels)', setup,