ENH: to_panel refactor and retool HDFStore table reader to not have to create MultiIndex, fixes perf regression

Author: wesm

pandas/core/frame.py

@@ -806,54 +806,30 @@ def to_panel(self):
         panel : Panel
         """
         from pandas.core.panel import Panel
-
-        wide_shape = (len(self.columns), len(self.index.levels[0]),
-                      len(self.index.levels[1]))
+        from pandas.core.reshape import block2d_to_block3d
 
         # only support this kind for now
         assert(isinstance(self.index, MultiIndex) and
                len(self.index.levels) == 2)
 
+        self._consolidate_inplace()
+
         major_axis, minor_axis = self.index.levels
+        major_labels, minor_labels = self.index.labels
 
-        def make_mask(index):
-            """
-            Create observation selection vector using major and minor
-            labels, for converting to wide format.
-            """
-            N, K = index.levshape
-            selector = index.labels[1] + K * index.labels[0]
-            mask = np.zeros(N * K, dtype=bool)
-            mask.put(selector, True)
-            return mask
-
-        def _to_wide_homogeneous():
-            values = np.empty(wide_shape, dtype=self.values.dtype)
-            if not issubclass(values.dtype.type, np.integer):
-                values.fill(np.nan)
-
-            frame_values = self.values
-            for i in xrange(len(self.columns)):
-                values[i].flat[mask] = frame_values[:, i]
-            return Panel(values, self.columns, major_axis, minor_axis)
-
-        def _to_wide_mixed():
-            _, N, K = wide_shape
-            # TODO: make much more efficient
-            data = {}
-            for item in self.columns:
-                item_vals = self[item].values
-                values = np.empty((N, K), dtype=item_vals.dtype)
-                values.flat[mask] = item_vals
-                data[item] = DataFrame(values, index=major_axis,
-                                       columns=minor_axis)
-            return Panel(data, self.columns, major_axis, minor_axis)
-
-        mask = make_mask(self.index)
-        if self._is_mixed_type:
-            return _to_wide_mixed()
-        else:
-            return _to_wide_homogeneous()
+        shape = len(major_axis), len(minor_axis)
+
+        new_blocks = []
+        for block in self._data.blocks:
+            newb = block2d_to_block3d(block.values.T, block.items, shape,
+                                      major_labels, minor_labels,
+                                      ref_items=self.columns)
+            new_blocks.append(newb)
+
+        new_axes = [self.columns, major_axis, minor_axis]
+        new_mgr = BlockManager(new_blocks, new_axes)
+
+        return Panel(new_mgr)
 
     to_wide = deprecate('to_wide', to_panel)
 

pandas/core/index.py

@@ -998,11 +998,26 @@ def __getitem__(self, key):
             return np.ndarray.__getitem__(self, key)
 
 def unique_with_labels(values):
-    uniques = Index(lib.fast_unique(values))
-    labels = lib.get_unique_labels(values, uniques.indexMap)
-    uniques._cleanup()
+    rizer = lib.Factorizer(len(values))
+    labels, _ = rizer.factorize(values, sort=False)
+    uniques = Index(rizer.uniques)
+
+    sorter = uniques.argsort()
+    reverse_indexer = np.empty(len(sorter), dtype='i4')
+    reverse_indexer.put(sorter, np.arange(len(sorter)))
+    labels = reverse_indexer.take(labels)
+    uniques = uniques.take(sorter)
+
     return uniques, labels
 
+def unique_int64(values):
+    if values.dtype != np.int64:
+        values = values.astype('i8')
+
+    table = lib.Int64HashTable(len(values))
+    uniques = table.unique(values)
+    return uniques
+
 class MultiIndex(Index):
     """
     Implements multi-level, a.k.a. hierarchical, index object for pandas objects

pandas/core/reshape.py

@@ -5,8 +5,9 @@
 
 import numpy as np
 
-from pandas.core.frame import DataFrame
 from pandas.core.series import Series
+from pandas.core.frame import DataFrame
+from pandas.core.panel import Panel
 
 from pandas.core.common import notnull
 from pandas.core.groupby import get_group_index
@@ -565,3 +566,33 @@ def make_axis_dummies(frame, axis='minor', transform=None):
     values = values.take(labels, axis=0)
 
     return DataFrame(values, columns=items, index=frame.index)
+
+def block2d_to_block3d(values, items, shape, major_labels, minor_labels,
+                       ref_items=None):
+    """
+    Developer method for pivoting DataFrame -> Panel. Used in HDFStore and
+    DataFrame.to_panel
+    """
+    from pandas.core.internals import make_block
+    panel_shape = (len(items),) + shape
+
+    # TODO: lexsort depth needs to be 2!!
+
+    # Create observation selection vector using major and minor
+    # labels, for converting to panel format.
+    selector = minor_labels + shape[1] * major_labels
+    mask = np.zeros(np.prod(shape), dtype=bool)
+    mask.put(selector, True)
+
+    pvalues = np.empty(panel_shape, dtype=values.dtype)
+    if not issubclass(pvalues.dtype.type, np.integer):
+        pvalues.fill(np.nan)
+
+    values = values
+    for i in xrange(len(items)):
+        pvalues[i].flat[mask] = values[:, i]
+
+    if ref_items is None:
+        ref_items = items
+
+    return make_block(pvalues, items, ref_items)

pandas/io/pytables.py

@@ -624,7 +624,10 @@ def _read_frame_table(self, group, where=None):
         return self._read_panel_table(group, where)['value']
 
     def _read_panel_table(self, group, where=None):
+        from pandas.core.index import unique_int64, Factor
         from pandas.core.common import _asarray_tuplesafe
+        from pandas.core.internals import BlockManager
+        from pandas.core.reshape import block2d_to_block3d
 
         table = getattr(group, 'table')
 
@@ -637,19 +640,37 @@ def _read_panel_table(self, group, where=None):
                                  table._v_attrs.columns_kind)
         index = _maybe_convert(sel.values['index'],
                                table._v_attrs.index_kind)
-        # reconstruct
-        long_index = MultiIndex.from_arrays([index, columns])
-        lp = DataFrame(sel.values['values'], index=long_index,
-                       columns=fields)
+        values = sel.values['values']
 
-        if not long_index.has_duplicates:
-            lp = lp.sortlevel(level=0)
-            wp = lp.to_panel()
+        major = Factor(index)
+        minor = Factor(columns)
+
+        J, K = len(major.levels), len(minor.levels)
+        key = major.labels * K + minor.labels
+
+        if len(unique_int64(key)) == len(key):
+            sorter, _ = lib.groupsort_indexer(key, J * K)
+
+            # the data need to be sorted
+            sorted_values = values.take(sorter, axis=0)
+            major_labels = major.labels.take(sorter)
+            minor_labels = minor.labels.take(sorter)
+
+            block = block2d_to_block3d(sorted_values, fields, (J, K),
+                                       major_labels, minor_labels)
+
+            mgr = BlockManager([block], [block.items,
+                                         major.levels, minor.levels])
+            wp = Panel(mgr)
         else:
             if not self._quiet:  # pragma: no cover
                 print ('Duplicate entries in table, taking most recently '
                        'appended')
 
+            # reconstruct
+            long_index = MultiIndex.from_arrays([index, columns])
+            lp = DataFrame(values, index=long_index, columns=fields)
+
             # need a better algorithm
             tuple_index = long_index.get_tuple_index()
             index_map = lib.map_indices_object(tuple_index)

pandas/src/hashtable.pyx

@@ -567,6 +567,8 @@ cdef class Int64HashTable:
             khiter_t k
             list uniques = []
 
+        # TODO: kvec
+
         for i in range(n):
             val = values[i]
             k = kh_get_int64(self.table, val)