Merge PR #2407

Author: wesm

RELEASE.rst

@@ -43,6 +43,11 @@ pandas 0.10.0
     `describe_option`, and `reset_option`. Deprecate `set_printoptions` and
     `reset_printoptions` (#2393)
 
+**Experimental Features**
+  - Add support for Panel4D, a named 4 Dimensional stucture
+  - Add support for ndpanel factory functions, to create custom, domain-specific
+     N Dimensional containers
+
 **API Changes**
 
   - The default binning/labeling behavior for ``resample`` has been changed to
@@ -91,7 +96,9 @@ pandas 0.10.0
     structures, which should do the right thing on both py2.x and py3.x. (#2224)
   - Reduce groupby.apply overhead substantially by low-level manipulation of
     internal NumPy arrays in DataFrames (#535)
-  - Implement ``value_vars`` in ``melt`` and add ``melt`` to pandas namespace (#2412)
+  - Implement ``value_vars`` in ``melt`` and add ``melt`` to pandas namespace
+    (#2412)
+  - Added boolean comparison operators to Panel
 
 **Bug fixes**
 

doc/source/dsintro.rst

@@ -798,3 +798,122 @@ method:
                  major_axis=date_range('1/1/2000', periods=5),
                  minor_axis=['a', 'b', 'c', 'd'])
    panel.to_frame()
+
+Panel4D (Experimental)
+----------------------
+
+``Panel4D`` is a 4-Dimensional named container very much like a ``Panel``, but
+having 4 named dimensions. It is intended as a test bed for more N-Dimensional named
+containers.
+
+  - **labels**: axis 0, each item corresponds to a Panel contained inside
+  - **items**: axis 1, each item corresponds to a DataFrame contained inside
+  - **major_axis**: axis 2, it is the **index** (rows) of each of the
+    DataFrames
+  - **minor_axis**: axis 3, it is the **columns** of each of the DataFrames
+
+
+``Panel4D`` is a sub-class of ``Panel``, so most methods that work on Panels are
+applicable to Panel4D. The following methods are disabled:
+
+  - ``join , to_frame , to_excel , to_sparse , groupby``
+
+Construction of Panel4D works in a very similar manner to a ``Panel``
+
+From 4D ndarray with optional axis labels
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. ipython:: python
+
+   p4d = Panel4D(randn(2, 2, 5, 4),
+              labels=['Label1','Label2'],
+              items=['Item1', 'Item2'],
+              major_axis=date_range('1/1/2000', periods=5),
+              minor_axis=['A', 'B', 'C', 'D'])
+   p4d
+
+
+From dict of Panel objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. ipython:: python
+
+   data = { 'Label1' : Panel({ 'Item1' : DataFrame(randn(4, 3)) }),
+            'Label2' : Panel({ 'Item2' : DataFrame(randn(4, 2)) }) }
+   Panel4D(data)
+
+Note that the values in the dict need only be **convertible to Panels**.
+Thus, they can be any of the other valid inputs to Panel as per above.
+
+Slicing
+~~~~~~~
+
+Slicing works in a similar manner to a Panel. ``[]`` slices the first dimension.
+``.ix`` allows you to slice abitrarily and get back lower dimensional objects
+
+.. ipython:: python
+
+   p4d['Label1']
+
+4D -> Panel
+
+.. ipython:: python
+
+   p4d.ix[:,:,:,'A']
+
+4D -> DataFrame
+
+.. ipython:: python
+
+   p4d.ix[:,:,0,'A']
+
+4D -> Series
+
+.. ipython:: python
+
+   p4d.ix[:,0,0,'A']
+
+Transposing
+~~~~~~~~~~~
+
+A Panel4D can be rearranged using its ``transpose`` method (which does not make a
+copy by default unless the data are heterogeneous):
+
+.. ipython:: python
+
+   p4d.transpose(3, 2, 1, 0)
+
+PanelND (Experimental)
+----------------------
+
+PanelND is a module with a set of factory functions to enable a user to construct N-dimensional named
+containers like Panel4D, with a custom set of axis labels. Thus a domain-specific container can easily be
+created.
+
+The following creates a Panel5D. A new panel type object must be sliceable into a lower dimensional object.
+Here we slice to a Panel4D.
+
+.. ipython:: python
+
+    from pandas.core import panelnd
+    Panel5D = panelnd.create_nd_panel_factory(
+        klass_name   = 'Panel5D',
+        axis_orders  = [ 'cool', 'labels','items','major_axis','minor_axis'],
+        axis_slices  = { 'labels' : 'labels', 'items' : 'items',
+	                 'major_axis' : 'major_axis', 'minor_axis' : 'minor_axis' },
+        slicer       = Panel4D,
+        axis_aliases = { 'major' : 'major_axis', 'minor' : 'minor_axis' },
+        stat_axis    = 2)
+    
+    p5d = Panel5D(dict(C1 = p4d))
+    p5d
+
+    # print a slice of our 5D
+    p5d.ix['C1',:,:,0:3,:]
+
+    # transpose it
+    p5d.transpose(1,2,3,4,0)
+
+    # look at the shape & dim
+    p5d.shape
+    p5d.ndim

doc/source/v0.10.0.txt

@@ -113,6 +113,23 @@ Updated PyTables Support
 
   - Implement ``value_vars`` in ``melt`` and add ``melt`` to pandas namespace (GH2412_)
 
+N Dimensional Panels (Experimental)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Adding experimental support for Panel4D and factory functions to create n-dimensional named panels.
+:ref:`Docs <dsintro-panel4d>` for NDim. Here is a taste of what to expect.
+
+     .. ipython:: python
+
+        p4d = Panel4D(randn(2, 2, 5, 4),
+              labels=['Label1','Label2'],
+              items=['Item1', 'Item2'],
+              major_axis=date_range('1/1/2000', periods=5),
+              minor_axis=['A', 'B', 'C', 'D'])
+        p4d
+
+
+
 API changes
 ~~~~~~~~~~~
 

pandas/core/panel.py

@@ -13,6 +13,7 @@
                                _get_combined_index)
 from pandas.core.indexing import _NDFrameIndexer, _maybe_droplevels
 from pandas.core.internals import BlockManager, make_block, form_blocks
+from pandas.core.series import Series
 from pandas.core.frame import DataFrame
 from pandas.core.generic import NDFrame
 from pandas.util import py3compat
@@ -104,7 +105,7 @@ def f(self, other):
 
 def _panel_arith_method(op, name):
     @Substitution(op)
-    def f(self, other, axis='items'):
+    def f(self, other, axis = 0):
         """
         Wrapper method for %s
 
@@ -123,6 +124,44 @@ def f(self, other, axis='items'):
     f.__name__ = name
     return f
 
+def _comp_method(func, name):
+
+    def na_op(x, y):
+        try:
+            result = func(x, y)
+        except TypeError:
+            xrav = x.ravel()
+            result = np.empty(x.size, dtype=x.dtype)
+            if isinstance(y, np.ndarray):
+                yrav = y.ravel()
+                mask = notnull(xrav) & notnull(yrav)
+                result[mask] = func(np.array(list(xrav[mask])),
+                                  np.array(list(yrav[mask])))
+            else:
+                mask = notnull(xrav)
+                result[mask] = func(np.array(list(xrav[mask])), y)
+
+            if func == operator.ne:  # pragma: no cover
+                np.putmask(result, -mask, True)
+            else:
+                np.putmask(result, -mask, False)
+            result = result.reshape(x.shape)
+
+        return result
+
+    @Appender('Wrapper for comparison method %s' % name)
+    def f(self, other):
+        if isinstance(other, self._constructor):
+            return self._compare_constructor(other, func)
+        elif isinstance(other, (self._constructor_sliced, DataFrame, Series)):
+            raise Exception("input needs alignment for this object [%s]" % self._constructor)
+        else:
+            return self._combine_const(other, na_op)
+
+
+    f.__name__ = name
+
+    return f
 
 _agg_doc = """
 Return %(desc)s over requested axis
@@ -280,7 +319,6 @@ def _init_dict(self, data, axes, dtype=None):
 
         # shallow copy
         arrays = []
-        reshaped_data = data.copy()
         haxis_shape = [ len(a) for a in raxes ]
         for h in haxis:
             v = values = data.get(h)
@@ -401,6 +439,51 @@ def __array_wrap__(self, result):
         d['copy'] = False
         return self._constructor(result, **d)
 
+    #----------------------------------------------------------------------
+    # Comparison methods
+
+    def _indexed_same(self, other):
+        return all([ getattr(self,a).equals(getattr(other,a)) for a in self._AXIS_ORDERS ])
+  
+    def _compare_constructor(self, other, func):
+        if not self._indexed_same(other):
+            raise Exception('Can only compare identically-labeled '
+                            'same type objects')
+
+        new_data = {}
+        for col in getattr(self,self._info_axis):
+            new_data[col] = func(self[col], other[col])
+
+        d = self._construct_axes_dict()
+        d['copy'] = False
+        return self._constructor(data=new_data, **d)
+
+    # boolean operators
+    __and__ = _arith_method(operator.and_, '__and__')
+    __or__ = _arith_method(operator.or_, '__or__')
+    __xor__ = _arith_method(operator.xor, '__xor__')
+
+    def __neg__(self):
+        return -1 * self
+
+    def __invert__(self):
+        return -1 * self
+    
+    # Comparison methods
+    __eq__ = _comp_method(operator.eq, '__eq__')
+    __ne__ = _comp_method(operator.ne, '__ne__')
+    __lt__ = _comp_method(operator.lt, '__lt__')
+    __gt__ = _comp_method(operator.gt, '__gt__')
+    __le__ = _comp_method(operator.le, '__le__')
+    __ge__ = _comp_method(operator.ge, '__ge__')
+
+    eq = _comp_method(operator.eq, 'eq')
+    ne = _comp_method(operator.ne, 'ne')
+    gt = _comp_method(operator.gt, 'gt')
+    lt = _comp_method(operator.lt, 'lt')
+    ge = _comp_method(operator.ge, 'ge')
+    le = _comp_method(operator.le, 'le')
+
     #----------------------------------------------------------------------
     # Magic methods
 
@@ -435,14 +518,14 @@ def __unicode__(self):
         class_name = str(self.__class__)
 
         shape = self.shape
-        dims = 'Dimensions: %s' % ' x '.join([ "%d (%s)" % (s, a) for a,s in zip(self._AXIS_ORDERS,shape) ])
+        dims = u'Dimensions: %s' % ' x '.join([ "%d (%s)" % (s, a) for a,s in zip(self._AXIS_ORDERS,shape) ])
 
         def axis_pretty(a):
             v = getattr(self,a)
             if len(v) > 0:
-                return '%s axis: %s to %s' % (a.capitalize(),v[0],v[-1])
+                return u'%s axis: %s to %s' % (a.capitalize(),com.pprint_thing(v[0]),com.pprint_thing(v[-1]))
             else:
-                return '%s axis: None' % a.capitalize()
+                return u'%s axis: None' % a.capitalize()
 
 
         output = '\n'.join([class_name, dims] + [axis_pretty(a) for a in self._AXIS_ORDERS])
@@ -496,9 +579,9 @@ def ix(self):
         return self._ix
 
     def _wrap_array(self, arr, axes, copy=False):
-        items, major, minor = axes
-        return self._constructor(arr, items=items, major_axis=major,
-                                 minor_axis=minor, copy=copy)
+        d    = dict([ (a,ax) for a,ax in zip(self._AXIS_ORDERS,axes) ])
+        d['copy'] = False
+        return self._constructor(arr, **d)
 
     fromDict = from_dict
 
@@ -742,7 +825,10 @@ def reindex(self, major=None, minor=None, method=None,
         if (method is None and not self._is_mixed_type and al <= 3):
             items = kwargs.get('items')
             if com._count_not_none(items, major, minor) == 3:
-                return self._reindex_multi(items, major, minor)
+                try:
+                    return self._reindex_multi(items, major, minor)
+                except:
+                    pass
 
         if major is not None:
             result = result._reindex_axis(major, method, al-2, copy)
@@ -874,12 +960,12 @@ def _combine(self, other, func, axis=0):
         elif isinstance(other, DataFrame):
             return self._combine_frame(other, func, axis=axis)
         elif np.isscalar(other):
-            new_values = func(self.values, other)
-            d = self._construct_axes_dict()
-            return self._constructor(new_values, **d)
+            return self._combine_const(other, func)
 
-    def __neg__(self):
-        return -1 * self
+    def _combine_const(self, other, func):
+        new_values = func(self.values, other)
+        d = self._construct_axes_dict()
+        return self._constructor(new_values, **d)
 
     def _combine_frame(self, other, func, axis=0):
         index, columns = self._get_plane_axes(axis)
@@ -1434,8 +1520,8 @@ def update(self, other, join='left', overwrite=True, filter_func=None,
             contain data in the same place.
         """
 
-        if not isinstance(other, Panel):
-            other = Panel(other)
+        if not isinstance(other, self._constructor):
+            other = self._constructor(other)
 
         other = other.reindex(items=self.items)