ENH: speed up DataFrame constructor with nested dict, GH #212

Author: wesm

RELEASE.rst

@@ -25,9 +25,13 @@ pandas 0.5.0
   - Added ability to join on multiple columns in `DataFrame.join` (GH #214)
   - Added private `_get_duplicates` function to `Index` for identifying
     duplicate values more easily
+  - Added column attribute access to DataFrame, e.g. df.A equivalent to df['A']
+    if 'A' is a column in the DataFrame (PR #213)
 
 **Improvements to existing features**
 
+  - Added Cython function for converting tuples to ndarray very fast. Speeds up
+    many MultiIndex-related operations
   - File parsing functions like `read_csv` and `read_table` will explicitly
     check if a parsed index has duplicates and raise a more helpful exception
     rather than deferring the check until later
@@ -38,6 +42,7 @@ pandas 0.5.0
     5x, regression from 0.3.0
   - With new `DataFrame.align` method, speeding up binary operations between
     differently-indexed DataFrame objects by 10-25%.
+  - Significantly sped up conversion of nested dict into DataFrame
 
 **Bug fixes**
 

pandas/core/frame.py

@@ -33,7 +33,7 @@
 from pandas.util import py3compat
 import pandas.core.common as common
 import pandas.core.datetools as datetools
-import pandas._tseries as _tseries
+import pandas._tseries as lib
 
 #----------------------------------------------------------------------
 # Factory helper methods
@@ -861,7 +861,7 @@ def _getitem_single(self, key):
         res = Series(values, index=self.index, name=key)
         self._series_cache[key] = res
         return res
-    
+
     def __getattr__(self, name):
         """After regular attribute access, try looking up the name of a column.
         This allows simpler access to columns for interactive use."""
@@ -3125,7 +3125,7 @@ def _get_index(v):
         if isinstance(v, Series):
             return v.index
         elif isinstance(v, dict):
-            return Index(_try_sort(v))
+            return v.keys()
 
     index = None
     if len(data) == 0:
@@ -3155,26 +3155,45 @@ def _get_index(v):
 
 def _union_indexes(indexes):
     if len(indexes) == 1:
-        index = indexes[0]
-    if _any_special_indexes(indexes):
+        result = indexes[0]
+        if isinstance(result, list):
+            result = Index(sorted(result))
+        return result
+
+    indexes, kind = _sanitize_and_check(indexes)
+
+    if kind == 'special':
         result = indexes[0]
         for other in indexes[1:]:
             result = result.union(other)
         return result
-    else:
+    elif kind == 'array':
         index = indexes[0]
         for other in indexes[1:]:
             if not index.equals(other):
-                return Index(_tseries.fast_unique_multiple(indexes))
+                return Index(lib.fast_unique_multiple(indexes))
 
         return index
+    else:
+        return Index(lib.fast_unique_multiple_list(indexes))
+
+
+def _sanitize_and_check(indexes):
+    kinds = list(set([type(index) for index in indexes]))
+
+    if list in kinds:
+        if len(kinds) > 1:
+            indexes = [Index(_try_sort(x)) if not isinstance(x, Index) else x
+                       for x in indexes]
+            kinds.remove(list)
+        else:
+            return indexes, 'list'
 
 
-def _any_special_indexes(indexes):
-    for index in indexes:
-        if type(index) != Index:
-            return True
-    return False
+    if len(kinds) > 1 or Index not in kinds:
+        return indexes, 'special'
+    else:
+        return indexes, 'array'
 
 
 def _check_data_types(data):

pandas/core/index.py

@@ -39,6 +39,7 @@ class Index(np.ndarray):
     ----
     An Index instance can **only** contain hashable objects
     """
+    name = None
     def __new__(cls, data, dtype=None, copy=False, name=None):
         if isinstance(data, np.ndarray):
             if dtype is None and issubclass(data.dtype.type, np.integer):
@@ -51,12 +52,6 @@ def __new__(cls, data, dtype=None, copy=False, name=None):
             # other iterable of some kind
             subarr = _asarray_tuplesafe(data, dtype=object)
 
-            # if not isinstance(data, (list, tuple)):
-            #     data = list(data)
-
-            # subarr = np.empty(len(data), dtype=object)
-            # subarr[:] = data
-
         subarr = subarr.view(cls)
         subarr.name = name
         return subarr

pandas/src/groupby.pyx

@@ -213,27 +213,6 @@ def group_labels2(ndarray[object] values):
 
     return reverse, labels
 
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def fast_unique(ndarray[object] values):
-    cdef:
-        Py_ssize_t i, n = len(values)
-        list uniques = []
-        dict table = {}
-        object val, stub = 0
-
-    for i from 0 <= i < n:
-        val = values[i]
-        if val not in table:
-            table[val] = stub
-            uniques.append(val)
-    try:
-        uniques.sort()
-    except Exception:
-        pass
-
-    return uniques
-
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def get_unique_labels(ndarray[object] values, dict idMap):
@@ -248,32 +227,6 @@ def get_unique_labels(ndarray[object] values, dict idMap):
 
     return fillVec
 
-@cython.wraparound(False)
-@cython.boundscheck(False)
-def fast_unique_multiple(list arrays):
-    cdef:
-        ndarray[object] buf
-        Py_ssize_t k = len(arrays)
-        Py_ssize_t i, j, n
-        list uniques = []
-        dict table = {}
-        object val, stub = 0
-
-    for i from 0 <= i < k:
-        buf = arrays[i]
-        n = len(buf)
-        for j from 0 <= j < n:
-            val = buf[j]
-            if val not in table:
-                table[val] = stub
-                uniques.append(val)
-    try:
-        uniques.sort()
-    except Exception:
-        pass
-
-    return uniques
-
 # from libcpp.set cimport set as stlset
 
 # cdef fast_unique_int32(ndarray arr):

pandas/src/tseries.pyx

@@ -274,6 +274,80 @@ def list_to_object_array(list obj):
 
     return arr
 
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def fast_unique(ndarray[object] values):
+    cdef:
+        Py_ssize_t i, n = len(values)
+        list uniques = []
+        dict table = {}
+        object val, stub = 0
+
+    for i from 0 <= i < n:
+        val = values[i]
+        if val not in table:
+            table[val] = stub
+            uniques.append(val)
+    try:
+        uniques.sort()
+    except Exception:
+        pass
+
+    return uniques
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def fast_unique_multiple(list arrays):
+    cdef:
+        ndarray[object] buf
+        Py_ssize_t k = len(arrays)
+        Py_ssize_t i, j, n
+        list uniques = []
+        dict table = {}
+        object val, stub = 0
+
+    for i from 0 <= i < k:
+        buf = arrays[i]
+        n = len(buf)
+        for j from 0 <= j < n:
+            val = buf[j]
+            if val not in table:
+                table[val] = stub
+                uniques.append(val)
+    try:
+        uniques.sort()
+    except Exception:
+        pass
+
+    return uniques
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def fast_unique_multiple_list(list lists):
+    cdef:
+        list buf
+        Py_ssize_t k = len(lists)
+        Py_ssize_t i, j, n
+        list uniques = []
+        dict table = {}
+        object val, stub = 0
+
+    for i from 0 <= i < k:
+        buf = lists[i]
+        n = len(buf)
+        for j from 0 <= j < n:
+            val = buf[j]
+            if val not in table:
+                table[val] = stub
+                uniques.append(val)
+    try:
+        uniques.sort()
+    except Exception:
+        pass
+
+    return uniques
+
 include "skiplist.pyx"
 include "groupby.pyx"
 include "moments.pyx"