REF: more refactoring for LongPanel removal

Author: wesm

pandas/core/frame.py

@@ -100,11 +100,11 @@ def _add_stat_doc(f, name, shortname, na_action=_doc_exclude_na,
     f.__doc__ = doc
 
 def _arith_method(func, name, default_axis='columns'):
-    def f(self, other, axis=default_axis, fill_value=None):
+    def f(self, other, axis=default_axis, level=None, fill_value=None):
         if isinstance(other, DataFrame):    # Another DataFrame
-            return self._combine_frame(other, func, fill_value)
+            return self._combine_frame(other, func, fill_value, level)
         elif isinstance(other, Series):
-            return self._combine_series(other, func, fill_value, axis)
+            return self._combine_series(other, func, fill_value, axis, level)
         else:
             return self._combine_const(other, func)
 
@@ -1258,7 +1258,7 @@ def xs(self, key, axis=0, copy=True):
     #----------------------------------------------------------------------
     # Reindexing and alignment
 
-    def align(self, other, join='outer', axis=None, copy=True):
+    def align(self, other, join='outer', axis=None, level=None, copy=True):
         """
         Align two DataFrame object on their index and columns with the specified
         join method for each axis Index
@@ -1276,13 +1276,16 @@ def align(self, other, join='outer', axis=None, copy=True):
             Aligned Series
         """
         if isinstance(other, DataFrame):
-            return self._align_frame(other, join=join, axis=axis, copy=copy)
+            return self._align_frame(other, join=join, axis=axis, level=level,
+                                     copy=copy)
         elif isinstance(other, Series):
-            return self._align_series(other, join=join, axis=axis, copy=copy)
+            return self._align_series(other, join=join, axis=axis, level=level,
+                                      copy=copy)
         else:  # pragma: no cover
             raise TypeError('unsupported type: %s' % type(other))
 
-    def _align_frame(self, other, join='outer', axis=None, copy=True):
+    def _align_frame(self, other, join='outer', axis=None, level=None,
+                     copy=True):
         # defaults
         join_index = self.index
         join_columns = self.columns
@@ -1291,14 +1294,14 @@ def _align_frame(self, other, join='outer', axis=None, copy=True):
 
         if axis is None or axis == 0:
             if not self.index.equals(other.index):
-                join_index, ilidx, iridx = self.index.join(other.index, how=join,
-                                                           return_indexers=True)
+                join_index, ilidx, iridx = \
+                    self.index.join(other.index, how=join, return_indexers=True)
 
         if axis is None or axis == 1:
             if not self.columns.equals(other.columns):
-                join_columns, clidx, cridx = self.columns.join(other.columns,
-                                                               how=join,
-                                                               return_indexers=True)
+                join_columns, clidx, cridx = \
+                    self.columns.join(other.columns, how=join,
+                                      return_indexers=True)
 
         def _align(frame, row_idx, col_idx):
             new_data = frame._data
@@ -1318,7 +1321,8 @@ def _align(frame, row_idx, col_idx):
         right = _align(other, iridx, cridx)
         return left, right
 
-    def _align_series(self, other, join='outer', axis=None, copy=True):
+    def _align_series(self, other, join='outer', axis=None, level=None,
+                      copy=True):
         fdata = self._data
         if axis == 0:
             join_index = self.index
@@ -1348,7 +1352,8 @@ def _align_series(self, other, join='outer', axis=None, copy=True):
         right_result = other if ridx is None else other.reindex(join_index)
         return left_result, right_result
 
-    def reindex(self, index=None, columns=None, method=None, copy=True):
+    def reindex(self, index=None, columns=None, method=None, level=None,
+                copy=True):
         """Conform Series to new index with optional filling logic, placing
         NA/NaN in locations having no value in the previous index. A new object
         is produced unless the new index is equivalent to the current one and
@@ -1381,15 +1386,15 @@ def reindex(self, index=None, columns=None, method=None, copy=True):
 
         if index is not None:
             index = _ensure_index(index)
-            frame = frame._reindex_index(index, method, copy)
+            frame = frame._reindex_index(index, method, copy, level)
 
         if columns is not None:
             columns = _ensure_index(columns)
-            frame = frame._reindex_columns(columns, copy)
+            frame = frame._reindex_columns(columns, copy, level)
 
         return frame
 
-    def _reindex_index(self, new_index, method, copy):
+    def _reindex_index(self, new_index, method, copy, level):
         if new_index.equals(self.index):
             if copy:
                 result = self.copy()
@@ -1916,8 +1921,8 @@ def _rename_columns_inplace(self, mapper):
     #----------------------------------------------------------------------
     # Arithmetic / combination related
 
-    def _combine_frame(self, other, func, fill_value=None):
-        this, other = self.align(other, join='outer', copy=False)
+    def _combine_frame(self, other, func, fill_value=None, level=None):
+        this, other = self.align(other, join='outer', level=level, copy=False)
         new_index, new_columns = this.index, this.columns
 
         this_vals = this.values
@@ -3654,6 +3659,16 @@ def _is_sequence(x):
     except Exception:
         return False
 
+def _align_level(frame, multi_index, level, axis=0):
+    levnum = multi_index._get_level_number(level)
+
+    data = frame.reindex(multi_index.levels[levnum], copy=False)._data
+
+    mgr_axis = 0 if axis == 1 else 1
+    new_data = data.reindex_indexer(multi_index, multi_index.labels[levnum],
+                                    axis=mgr_axis)
+    return DataFrame(new_data)
+
 def install_ipython_completers():  # pragma: no cover
     """Register the DataFrame type with IPython's tab completion machinery, so
     that it knows about accessing column names as attributes."""

pandas/core/internals.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 import itertools
 
 from numpy import nan
@@ -685,6 +686,17 @@ def reindex_indexer(self, new_axis, indexer, axis=1):
 
         return BlockManager(new_blocks, new_axes)
 
+    def _reindex_indexer_items(new_axis, indexer):
+        from collections import defaultdict
+
+        dtypes = self.item_dtypes
+        result_dtypes = dtypes.take(indexer)
+        counts = defaultdict(int)
+        for t in result_dtypes:
+            counts[t] += 1
+
+
+
     def reindex_items(self, new_items):
         """
 
@@ -871,6 +883,17 @@ def block_id_vector(self):
         assert((result >= 0).all())
         return result
 
+    @property
+    def item_dtypes(self):
+        result = np.empty(len(self.items), dtype='O')
+        mask = np.zeros(len(self.items), dtype=bool)
+        for i, blk in enumerate(self.blocks):
+            indexer = self.items.get_indexer(blk.items)
+            result.put(indexer, blk.values.dtype.name)
+            mask.put(indexer, 1)
+        assert(mask.all())
+        return result
+
 def form_blocks(data, axes):
     # pre-filter out items if we passed it
     items = axes[0]
@@ -1242,3 +1265,16 @@ def _upcast_blocks(blocks):
         # use any ref_items
         return _consolidate(new_blocks, newb.ref_items)
 
+def _make_block_indexers(blocks, indexer, block_ids, block_locs, block_dtypes,
+                         ref_items):
+    counts = defaultdict(int)
+    for dtype_name in block_dtypes.take(indexer):
+        counts[dtype_name] += 1
+
+    findexer = np.empty(counts['float64'], dtype='i4')
+    bindexer = np.empty(counts['bool'], dtype='i4')
+    oindexer = np.empty(counts['object'], dtype='i4')
+    iindexer = np.empty(counts['int64'], dtype='i4')
+
+    for idx in indexer:
+        pass

pandas/io/pytables.py

@@ -9,8 +9,7 @@
 import time
 
 import numpy as np
-from pandas import (Series, TimeSeries, DataFrame, Panel, LongPanel,
-                    Index, MultiIndex)
+from pandas import Series, TimeSeries, DataFrame, Panel, Index, MultiIndex
 from pandas.core.common import adjoin
 import pandas.core.common as com
 import pandas._tseries as lib
@@ -20,8 +19,7 @@
     Series     : 'series',
     TimeSeries : 'series',
     DataFrame  : 'frame',
-    Panel  : 'wide',
-    LongPanel  : 'long'
+    Panel  : 'wide'
 }
 
 _NAME_MAP = {
@@ -32,7 +30,6 @@
     'wide' : 'Panel',
     'wide_table' : 'Panel (Table)',
     'long' : 'LongPanel',
-
     # legacy h5 files
     'Series' : 'Series',
     'TimeSeries' : 'TimeSeries',
@@ -244,7 +241,7 @@ def put(self, key, value, table=False, append=False,
         Parameters
         ----------
         key : object
-        value : {Series, DataFrame, Panel, LongPanel}
+        value : {Series, DataFrame, Panel}
         table : boolean, default False
             Write as a PyTables Table structure which may perform worse but
             allow more flexible operations like searching / selecting subsets of
@@ -294,7 +291,7 @@ def append(self, key, value):
         Parameters
         ----------
         key : object
-        value : {Series, DataFrame, Panel, LongPanel}
+        value : {Series, DataFrame, Panel}
 
         Notes
         -----
@@ -340,6 +337,22 @@ def _write_frame(self, group, df):
     def _read_frame(self, group, where=None):
         return DataFrame(self._read_block_manager(group))
 
+    def _write_long(self, group, panel):
+        if len(panel.values) == 0:
+            raise ValueError('Can not write empty structure, data length was 0')
+        self._write_block_manager(group, panel._data)
+
+    def _read_long(self, group, where=None):
+        items = self._read_index(group, 'items')
+        major_axis = self._read_index(group, 'major_axis')
+        minor_axis = self._read_index(group, 'minor_axis')
+        major_labels = _read_array(group, 'major_labels')
+        minor_labels = _read_array(group, 'minor_labels')
+        values = _read_array(group, 'values')
+        index = MultiIndex(levels=[major_axis, minor_axis],
+                           labels=[major_labels, minor_labels])
+        return DataFrame(values, index=index, columns=items)
+
     def _write_block_manager(self, group, data):
         if not data.is_consolidated():
             data = data.consolidate()
@@ -404,31 +417,6 @@ def _write_wide_table(self, group, panel, append=False, comp=None):
     def _read_wide_table(self, group, where=None):
         return self._read_panel_table(group, where)
 
-    def _write_long(self, group, panel, append=False):
-        if len(panel.values) == 0:
-            raise ValueError('Can not write empty structure, data length was 0')
-
-        self._write_index(group, 'major_axis', panel.major_axis)
-        self._write_index(group, 'minor_axis', panel.minor_axis)
-        self._write_index(group, 'items', panel.items)
-        self._write_array(group, 'major_labels', panel.major_labels)
-        self._write_array(group, 'minor_labels', panel.minor_labels)
-        self._write_array(group, 'values', panel.values)
-
-    def _read_long(self, group, where=None):
-        from pandas.core.index import MultiIndex
-
-        items = self._read_index(group, 'items')
-        major_axis = self._read_index(group, 'major_axis')
-        minor_axis = self._read_index(group, 'minor_axis')
-        major_labels = _read_array(group, 'major_labels')
-        minor_labels = _read_array(group, 'minor_labels')
-        values = _read_array(group, 'values')
-
-        index = MultiIndex(levels=[major_axis, minor_axis],
-                           labels=[major_labels, minor_labels])
-        return LongPanel(values, index=index, columns=items)
-
     def _write_index(self, group, key, index):
         if len(index) == 0:
             raise ValueError('Can not write empty structure, axis length was 0')

pandas/tests/test_multilevel.py

@@ -54,6 +54,9 @@ def test_append(self):
         result = a['A'].append(b['A'])
         tm.assert_series_equal(result, self.frame['A'])
 
+    def test_align_level(self):
+        pass
+
     def test_pickle(self):
         import cPickle
         def _test_roundtrip(frame):