BLD: py3 compat

TST: removed pytest in favor of nosetest for tests/test_msgpack

Author: jreback

doc/source/io.rst

@@ -36,7 +36,7 @@ object.
     * ``read_hdf``
     * ``read_sql``
     * ``read_json``
-    * ``read_msgpack``
+    * ``read_msgpack`` (experimental)
     * ``read_html``
     * ``read_stata``
     * ``read_clipboard``
@@ -49,7 +49,7 @@ The corresponding ``writer`` functions are object methods that are accessed like
     * ``to_hdf``
     * ``to_sql``
     * ``to_json``
-    * ``to_msgpack``
+    * ``to_msgpack`` (experimental)
     * ``to_html``
     * ``to_stata``
     * ``to_clipboard``
@@ -1737,23 +1737,23 @@ module is installed you can use it as a xlsx writer engine as follows:
 Serialization
 -------------
 
-msgpack
-~~~~~~~
+msgpack (experimental)
+~~~~~~~~~~~~~~~~~~~~~~
 
 .. _io.msgpack:
 
-.. versionadded:: 0.11.1
+.. versionadded:: 0.13.0
 
-Starting in 0.11.1, pandas is supporting the ``msgpack`` format for 
+Starting in 0.13.0, pandas is supporting the ``msgpack`` format for
 object serialization. This is a lightweight portable binary format, similar
-to binary JSON, that is highly space efficient, and provides good performance 
+to binary JSON, that is highly space efficient, and provides good performance
 both on the writing (serialization), and reading (deserialization).
 
 .. warning::
 
-   This is a very new feature of pandas. We intend to provide certain 
+   This is a very new feature of pandas. We intend to provide certain
    optimizations in the io of the ``msgpack`` data. We do not intend this
-   format to change (and will be backward compatible if we do).
+   format to change (however it is experimental)
 
 .. ipython:: python
 

doc/source/v0.13.0.txt

@@ -464,6 +464,15 @@ Enhancements
         t = Timestamp('20130101 09:01:02')
         t + pd.datetools.Nano(123)
 
+  - The ``isin`` method plays nicely with boolean indexing. To get the rows where each condition is met:
+
+      .. ipython:: python
+
+         mask = df.isin({'A': [1, 2], 'B': ['e', 'f']})
+         df[mask.all(1)]
+
+      See the :ref:`documentation<indexing.basics.indexing_isin>` for more.
+
 .. _whatsnew_0130.experimental:
 
 Experimental
@@ -553,21 +562,35 @@ Experimental
     For more details see the :ref:`indexing documentation on query
     <indexing.query>`.
 
-  - DataFrame now has an ``isin`` method that can be used to easily check whether the DataFrame's values are contained in an iterable.  Use a dictionary if you'd like to check specific iterables for specific columns or rows.
+- ``pd.read_msgpack()`` and ``pd.to_msgpack()`` are now a supported method of serialization
+  of arbitrary pandas (and python objects) in a lightweight portable binary format. :ref:`See the docs<io.msgpack>`
 
-    .. ipython:: python
+  .. warning::
 
-      df = pd.DataFrame({'A': [1, 2, 3], 'B': ['d', 'e', 'f']})
-      df.isin({'A': [1, 2], 'B': ['e', 'f']})
+     Since this is EXPERIMENTAL LIBRARY, the storage format may not be stable until a future release.
 
-  The ``isin`` method plays nicely with boolean indexing. To get the rows where each condition is met:
+  .. ipython:: python
 
-    .. ipython:: python
+     df = DataFrame(np.random.rand(5,2),columns=list('AB'))
+     df.to_msgpack('foo.msg')
+     pd.read_msgpack('foo.msg')
+
+     s = Series(np.random.rand(5),index=date_range('20130101',periods=5))
+     pd.to_msgpack('foo.msg', df, s)
+     pd.read_msgpack('foo.msg')
+
+  You can pass ``iterator=True`` to iterator over the unpacked results
 
-      mask = df.isin({'A': [1, 2], 'B': ['e', 'f']})
-      df[mask.all(1)]
+  .. ipython:: python
+
+     for o in pd.read_msgpack('foo.msg',iterator=True):
+        print o
+
+  .. ipython:: python
+     :suppress:
+     :okexcept:
 
-  See the :ref:`documentation<indexing.basics.indexing_isin>` for more.
+     os.remove('foo.msg')
 
 .. _whatsnew_0130.refactoring:
 
@@ -686,35 +709,6 @@ to unify methods and behaviors. Series formerly subclassed directly from
      s.a = 5
      s
 
-IO Enhancements
-~~~~~~~~~~~~~~~
-
-- ``pd.read_msgpack()`` and ``pd.to_msgpack()`` are now a supported method of serialization
-  of arbitrary pandas (and python objects) in a lightweight portable binary format. :ref:`See the docs<io.msgpack>`
-
-  .. ipython:: python
-
-        df = DataFrame(np.random.rand(5,2),columns=list('AB'))
-        df.to_msgpack('foo.msg')
-        pd.read_msgpack('foo.msg')
-
-        s = Series(np.random.rand(5),index=date_range('20130101',periods=5))
-        pd.to_msgpack('foo.msg', df, s)
-        pd.read_msgpack('foo.msg')
-
-  You can pass ``iterator=True`` to iterator over the unpacked results
-
-  .. ipython:: python
-
-        for o in pd.read_msgpack('foo.msg',iterator=True):
-            print o
-
-  .. ipython:: python
-        :suppress:
-        :okexcept:
-
-        os.remove('foo.msg')
-
 Bug Fixes
 ~~~~~~~~~
 

pandas/core/generic.py

@@ -806,6 +806,21 @@ def to_hdf(self, path_or_buf, key, **kwargs):
         return pytables.to_hdf(path_or_buf, key, self, **kwargs)
 
     def to_msgpack(self, path_or_buf, **kwargs):
+        """
+        msgpack (serialize) object to input file path
+
+        THIS IS AN EXPERIMENTAL LIBRARY and the storage format
+        may not be stable until a future release.
+
+        Parameters
+        ----------
+        path : string File path
+        args : an object or objects to serialize
+        append : boolean whether to append to an existing msgpack
+                (default is False)
+        compress : type of compressor (zlib or blosc), default to None (no compression)
+        """
+
         from pandas.io import packers
         return packers.to_msgpack(path_or_buf, self, **kwargs)
 

pandas/io/packers.py

@@ -49,9 +49,11 @@
 from dateutil.parser import parse
 
 import numpy as np
+from pandas import compat
+from pandas.compat import u
 from pandas import (
     Timestamp, Period, Series, TimeSeries, DataFrame, Panel, Panel4D,
-    Index, MultiIndex, Int64Index, PeriodIndex, DatetimeIndex, NaT
+    Index, MultiIndex, Int64Index, PeriodIndex, DatetimeIndex, Float64Index, NaT
 )
 from pandas.sparse.api import SparseSeries, SparseDataFrame, SparsePanel
 from pandas.sparse.array import BlockIndex, IntIndex
@@ -80,12 +82,13 @@ def to_msgpack(path, *args, **kwargs):
     """
     msgpack (serialize) object to input file path
 
+    THIS IS AN EXPERIMENTAL LIBRARY and the storage format
+    may not be stable until a future release.
+
     Parameters
     ----------
-    path : string
-        File path
+    path : string File path
     args : an object or objects to serialize
-
     append : boolean whether to append to an existing msgpack
              (default is False)
     compress : type of compressor (zlib or blosc), default to None (no compression)
@@ -112,6 +115,9 @@ def read_msgpack(path, iterator=False, **kwargs):
     Load msgpack pandas object from the specified
     file path
 
+    THIS IS AN EXPERIMENTAL LIBRARY and the storage format
+    may not be stable until a future release.
+
     Parameters
     ----------
     path : string
@@ -134,11 +140,11 @@ def read_msgpack(path, iterator=False, **kwargs):
         return l
 
 dtype_dict = { 21 : np.dtype('M8[ns]'),
-               u'datetime64[ns]' : np.dtype('M8[ns]'),
-               u'datetime64[us]' : np.dtype('M8[us]'),
+               u('datetime64[ns]') : np.dtype('M8[ns]'),
+               u('datetime64[us]') : np.dtype('M8[us]'),
                22 : np.dtype('m8[ns]'),
-               u'timedelta64[ns]' : np.dtype('m8[ns]'),
-               u'timedelta64[us]' : np.dtype('m8[us]') }
+               u('timedelta64[ns]') : np.dtype('m8[ns]'),
+               u('timedelta64[us]') : np.dtype('m8[us]') }
 
 def dtype_for(t):
     if t in dtype_dict:
@@ -157,7 +163,7 @@ def c2f(r, i, ctype_name):
     """
     Convert strings to complex number instance with specified numpy type.
     """
-    
+
     ftype = c2f_dict[ctype_name]
     return np.typeDict[ctype_name](ftype(r)+1j*ftype(i))
 
@@ -224,7 +230,7 @@ def encode(obj):
     """
     Data encoder
     """
-        
+
     tobj = type(obj)
     if isinstance(obj, Index):
         if isinstance(obj, PeriodIndex):
@@ -281,15 +287,15 @@ def encode(obj):
                  'columns' : obj.columns }
             for f in ['default_fill_value','default_kind']:
                 d[f] = getattr(obj,f,None)
-            d['data'] = dict([ (name,ss) for name,ss in obj.iterkv() ])
+            d['data'] = dict([ (name,ss) for name,ss in compat.iteritems(obj) ])
             return d
         elif isinstance(obj, SparsePanel):
             d = {'typ' : 'sparse_panel',
                  'klass' : obj.__class__.__name__,
                  'items' : obj.items }
             for f in ['default_fill_value','default_kind']:
                 d[f] = getattr(obj,f,None)
-            d['data'] = dict([ (name,df) for name,df in obj.iterkv() ])
+            d['data'] = dict([ (name,df) for name,df in compat.iteritems(obj) ])
             return d
         else:
 
@@ -301,8 +307,8 @@ def encode(obj):
             return {'typ' : 'block_manager',
                     'klass'  : obj.__class__.__name__,
                     'axes'   : data.axes,
-                    'blocks' : [ { 'items'  : b.items, 
-                                   'values' : convert(b.values), 
+                    'blocks' : [ { 'items'  : b.items,
+                                   'values' : convert(b.values),
                                    'shape'  : b.values.shape,
                                    'dtype'  : b.dtype.num,
                                    'klass' : b.__class__.__name__,
@@ -381,7 +387,7 @@ def decode(obj):
     """
     Decoder for deserializing numpy data types.
     """
-    
+
     typ = obj.get('typ')
     if typ is None:
         return obj
@@ -408,7 +414,7 @@ def decode(obj):
 
         def create_block(b):
             dtype = dtype_for(b['dtype'])
-            return make_block(unconvert(b['values'],dtype,b['compress']).reshape(b['shape']),b['items'],axes[0],klass=getattr(internals,b['klass'])) 
+            return make_block(unconvert(b['values'],dtype,b['compress']).reshape(b['shape']),b['items'],axes[0],klass=getattr(internals,b['klass']))
 
         blocks = [ create_block(b) for b in obj['blocks'] ]
         return globals()[obj['klass']](BlockManager(blocks, axes))
@@ -454,17 +460,17 @@ def create_block(b):
     else:
         return obj
 
-def pack(o, default=encode, 
+def pack(o, default=encode,
          encoding='utf-8', unicode_errors='strict', use_single_float=False):
     """
     Pack an object and return the packed bytes.
     """
 
     return Packer(default=default, encoding=encoding,
-           unicode_errors=unicode_errors, 
+           unicode_errors=unicode_errors,
            use_single_float=use_single_float).pack(o)
 
-def unpack(packed, object_hook=decode, 
+def unpack(packed, object_hook=decode,
            list_hook=None, use_list=False, encoding='utf-8',
            unicode_errors='strict', object_pairs_hook=None):
     """
@@ -473,17 +479,17 @@ def unpack(packed, object_hook=decode,
     """
 
     return Unpacker(packed, object_hook=object_hook,
-                    list_hook=list_hook, 
+                    list_hook=list_hook,
                     use_list=use_list, encoding=encoding,
-                    unicode_errors=unicode_errors, 
+                    unicode_errors=unicode_errors,
                     object_pairs_hook=object_pairs_hook)
 
 class Packer(_Packer):
-    def __init__(self, default=encode, 
+    def __init__(self, default=encode,
                  encoding='utf-8',
                  unicode_errors='strict',
                  use_single_float=False):
-        super(Packer, self).__init__(default=default, 
+        super(Packer, self).__init__(default=default,
                                      encoding=encoding,
                                      unicode_errors=unicode_errors,
                                      use_single_float=use_single_float)
@@ -493,14 +499,14 @@ def __init__(self, file_like=None, read_size=0, use_list=False,
                  object_hook=decode,
                  object_pairs_hook=None, list_hook=None, encoding='utf-8',
                  unicode_errors='strict', max_buffer_size=0):
-        super(Unpacker, self).__init__(file_like=file_like, 
-                                       read_size=read_size,    
-                                       use_list=use_list, 
-                                       object_hook=object_hook, 
-                                       object_pairs_hook=object_pairs_hook, 
+        super(Unpacker, self).__init__(file_like=file_like,
+                                       read_size=read_size,
+                                       use_list=use_list,
+                                       object_hook=object_hook,
+                                       object_pairs_hook=object_pairs_hook,
                                        list_hook=list_hook,
-                                       encoding=encoding, 
-                                       unicode_errors=unicode_errors, 
+                                       encoding=encoding,
+                                       unicode_errors=unicode_errors,
                                        max_buffer_size=max_buffer_size)
 
 class Iterator(object):

pandas/io/tests/test_packers.py

@@ -7,6 +7,8 @@
 import datetime
 import numpy as np
 
+from pandas import compat
+from pandas.compat import u
 from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
                     date_range, period_range, Index, SparseSeries, SparseDataFrame,
                     SparsePanel)
@@ -146,7 +148,7 @@ def test_numpy_array_complex(self):
                        x.dtype == x_rec.dtype)
 
     def test_list_mixed(self):
-        x = [1.0, np.float32(3.5), np.complex128(4.25), u'foo']
+        x = [1.0, np.float32(3.5), np.complex128(4.25), u('foo')]
         x_rec = self.encode_decode(x)
         self.assert_(all(map(lambda x,y: x == y, x, x_rec)) and \
                            all(map(lambda x,y: type(x) == type(y), x, x_rec)))

pandas/msgpack.pyx

@@ -110,7 +110,7 @@ cdef class Packer(object):
     * *defaut* - Convert user type to builtin type that Packer supports.
       See also simplejson's document.
     * *encoding* - Convert unicode to bytes with this encoding. (default: 'utf-8')
-    * *unicode_erros* - Error handler for encoding unicode. (default: 'strict')
+    * *unicode_errors* - Error handler for encoding unicode. (default: 'strict')
     * *use_single_float* - Use single precision float type for float. (default: False)
     * *autoreset* - Reset buffer after each pack and return it's content as `bytes`. (default: True).
       If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
@@ -242,7 +242,7 @@ cdef class Packer(object):
                     if ret != 0: break
 
         # ndarray support ONLY (and float64/int64) for now
-        elif isinstance(o, np.ndarray) and not hasattr(o,'values') and (o.dtype == 'float64' or o.dtype == 'int64'): 
+        elif isinstance(o, np.ndarray) and not hasattr(o,'values') and (o.dtype == 'float64' or o.dtype == 'int64'):
 
             ret = msgpack_pack_map(&self.pk, 5)
             if ret != 0: return -1
@@ -276,7 +276,7 @@ cdef class Packer(object):
                 for i in range(n):
 
                    i8val = array_int[i]
-                   ret = msgpack_pack_long(&self.pk, i8val)
+                   ret = msgpack_pack_long_long(&self.pk, i8val)
                    if ret != 0: break
 
         elif self._default: