ENH: added py3 pickles, revsied to include SparseSeries/SparseDataFrame/Index/MultiIndex

Author: jreback

pandas/io/tests/data/legacy_pickle/0.10.1/AMD64_windows_2.7.3.pickle

@@ -1,5 +1,37 @@
 """ self-contained to write legacy pickle files """
 
+def _create_sp_series():
+
+    import numpy as np
+    from pandas import bdate_range, SparseSeries
+
+    nan = np.nan
+
+    # nan-based
+    arr = np.arange(15, dtype=float)
+    index = np.arange(15)
+    arr[7:12] = nan
+    arr[-1:] = nan
+
+    date_index = bdate_range('1/1/2011', periods=len(index))
+    bseries = SparseSeries(arr, index=index, kind='block')
+    bseries.name = 'bseries'
+    return bseries
+
+def _create_sp_frame():
+    import numpy as np
+    from pandas import bdate_range, SparseDataFrame
+
+    nan = np.nan
+
+    data = {'A': [nan, nan, nan, 0, 1, 2, 3, 4, 5, 6],
+            'B': [0, 1, 2, nan, nan, nan, 3, 4, 5, 6],
+            'C': np.arange(10),
+            'D': [0, 1, 2, 3, 4, 5, nan, nan, nan, nan]}
+    
+    dates = bdate_range('1/1/2011', periods=10)
+    return SparseDataFrame(data, index=dates)
+
 def create_data():
     """ create the pickle data """
 
@@ -8,7 +40,8 @@ def create_data():
     from pandas import (Series,DataFrame,Panel,
                         SparseSeries,SparseDataFrame,SparsePanel,
                         Index,MultiIndex,PeriodIndex,
-                        date_range,Timestamp)
+                        date_range,bdate_range,Timestamp)
+    nan = np.nan
 
     data = {
         'A': [0., 1., 2., 3., np.nan],
@@ -18,17 +51,29 @@ def create_data():
         'E' : [0., 1, Timestamp('20100101'),'foo',2.],
         }
 
+    index  = dict(int   = Index(np.arange(10)),
+                  date  = date_range('20130101',periods=10))
+    mi     = dict(reg   = MultiIndex.from_tuples(zip([['bar', 'bar', 'baz', 'baz', 'foo', 'foo', 'qux', 'qux'],
+                                                      ['one', 'two', 'one', 'two', 'one', 'two', 'one', 'two']]),
+                                                 names=['first', 'second']))
     series = dict(float = Series(data['A']),
                   int   = Series(data['B']),
                   mixed = Series(data['E']))
     frame  = dict(float = DataFrame(dict(A = series['float'], B = series['float'] + 1)),
                   int   = DataFrame(dict(A = series['int']  , B = series['int']   + 1)),
                   mixed = DataFrame(dict([ (k,data[k]) for k in ['A','B','C','D']])))
     panel  = dict(float = Panel(dict(ItemA = frame['float'], ItemB = frame['float']+1)))
-    
+
+ 
+
     return dict( series = series, 
                  frame  = frame, 
-                 panel  = panel )
+                 panel  = panel,
+                 index  = index,
+                 mi     = mi,
+                 sp_series = dict(float = _create_sp_series()),
+                 sp_frame  = dict(float = _create_sp_frame())
+                 )
 
 def write_legacy_pickles():
 
@@ -43,7 +88,7 @@ def write_legacy_pickles():
     import platform as pl
     import cPickle as pickle
 
-    print "This script generates a pickle file for the current arch, system, and python version"
+    print("This script generates a pickle file for the current arch, system, and python version")
 
     base_dir, _ = os.path.split(os.path.abspath(__file__))
     base_dir = os.path.join(base_dir,'data/legacy_pickle')
@@ -68,7 +113,7 @@ def write_legacy_pickles():
     pickle.dump(create_data(),fh,pickle.HIGHEST_PROTOCOL)
     fh.close()
 
-    print "created pickle file: %s" % pth
+    print("created pickle file: %s" % pth)
 
 if __name__ == '__main__':
     write_legacy_pickles()

pandas/io/tests/data/legacy_pickle/0.10.1/x86_64_linux_2.7.3.pickle

@@ -12,6 +12,8 @@
 import numpy as np
 import pandas.util.testing as tm
 import pandas as pd
+from pandas import Index
+from pandas.sparse.tests import test_sparse
 
 class TestPickle(unittest.TestCase):
     _multiprocess_can_split_ = True
@@ -23,9 +25,14 @@ def setUp(self):
     def compare(self, vf):
 
         # py3 compat when reading py2 pickle
+        
         try:
             with open(vf,'rb') as fh:
                 data = pickle.load(fh)
+        except (ValueError):
+
+            # we are trying to read a py3 pickle in py2.....
+            return
         except:
             with open(vf,'rb') as fh:
                 data = pickle.load(fh, encoding='latin1')
@@ -35,8 +42,16 @@ def compare(self, vf):
 
                 expected = self.data[typ][dt]
 
-                comparator = getattr(tm,"assert_%s_equal" % typ)
-                comparator(result,expected)
+                if isinstance(expected,Index):
+                    self.assert_(expected.equals(result))
+                    continue
+
+                if typ.startswith('sp_'):
+                    comparator = getattr(test_sparse,"assert_%s_equal" % typ)
+                    comparator(result,expected,exact_indices=False)
+                else:
+                    comparator = getattr(tm,"assert_%s_equal" % typ)
+                    comparator(result,expected)
 
     def test_read_pickles_0_10_1(self):
 

pandas/io/tests/data/legacy_pickle/0.11.0/x86_64_linux_2.7.3.pickle

@@ -74,8 +74,7 @@ def _test_data2_zero():
     arr[np.isnan(arr)] = 0
     return arr, index
 
-
-def assert_sp_series_equal(a, b):
+def assert_sp_series_equal(a, b, exact_indices=True):
     assert(a.index.equals(b.index))
     assert_sp_array_equal(a, b)