41
41
_ZLIB_INSTALLED = True
42
42
43
43
44
+ @pytest .fixture (scope = 'module' )
45
+ def current_packers_data ():
46
+ # our current version packers data
47
+ from pandas .tests .io .generate_legacy_storage_files import (
48
+ create_msgpack_data )
49
+ return create_msgpack_data ()
50
+
51
+
52
+ @pytest .fixture (scope = 'module' )
53
+ def all_packers_data ():
54
+ # our all of our current version packers data
55
+ from pandas .tests .io .generate_legacy_storage_files import (
56
+ create_data )
57
+ return create_data ()
58
+
59
+
44
60
def check_arbitrary (a , b ):
45
61
46
62
if isinstance (a , (list , tuple )) and isinstance (b , (list , tuple )):
@@ -778,7 +794,16 @@ def test_default_encoding(self):
778
794
assert_frame_equal (result , frame )
779
795
780
796
781
- class TestMsgpack ():
797
+ def legacy_packers_versions ():
798
+ # yield the packers versions
799
+ path = tm .get_data_path ('legacy_msgpack' )
800
+ for v in os .listdir (path ):
801
+ p = os .path .join (path , v )
802
+ if os .path .isdir (p ):
803
+ yield v
804
+
805
+
806
+ class TestMsgpack (object ):
782
807
"""
783
808
How to add msgpack tests:
784
809
@@ -788,48 +813,38 @@ class TestMsgpack():
788
813
$ python generate_legacy_storage_files.py <output_dir> msgpack
789
814
790
815
3. Move the created pickle to "data/legacy_msgpack/<version>" directory.
791
-
792
- NOTE: TestMsgpack can't be a subclass of tm.Testcase to use test generator.
793
- http://stackoverflow.com/questions/6689537/nose-test-generators-inside-class
794
816
"""
795
817
796
- @classmethod
797
- def setup_class (cls ):
798
- from pandas .tests .io .generate_legacy_storage_files import (
799
- create_msgpack_data , create_data )
800
- cls .data = create_msgpack_data ()
801
- cls .all_data = create_data ()
802
- cls .path = u ('__%s__.msgpack' % tm .rands (10 ))
803
- cls .minimum_structure = {'series' : ['float' , 'int' , 'mixed' ,
804
- 'ts' , 'mi' , 'dup' ],
805
- 'frame' : ['float' , 'int' , 'mixed' , 'mi' ],
806
- 'panel' : ['float' ],
807
- 'index' : ['int' , 'date' , 'period' ],
808
- 'mi' : ['reg2' ]}
809
-
810
- def check_min_structure (self , data ):
818
+ minimum_structure = {'series' : ['float' , 'int' , 'mixed' ,
819
+ 'ts' , 'mi' , 'dup' ],
820
+ 'frame' : ['float' , 'int' , 'mixed' , 'mi' ],
821
+ 'panel' : ['float' ],
822
+ 'index' : ['int' , 'date' , 'period' ],
823
+ 'mi' : ['reg2' ]}
824
+
825
+ def check_min_structure (self , data , version ):
811
826
for typ , v in self .minimum_structure .items ():
812
827
assert typ in data , '"{0}" not found in unpacked data' .format (typ )
813
828
for kind in v :
814
829
msg = '"{0}" not found in data["{1}"]' .format (kind , typ )
815
830
assert kind in data [typ ], msg
816
831
817
- def compare (self , vf , version ):
832
+ def compare (self , current_data , all_data , vf , version ):
818
833
# GH12277 encoding default used to be latin-1, now utf-8
819
834
if LooseVersion (version ) < '0.18.0' :
820
835
data = read_msgpack (vf , encoding = 'latin-1' )
821
836
else :
822
837
data = read_msgpack (vf )
823
- self .check_min_structure (data )
838
+ self .check_min_structure (data , version )
824
839
for typ , dv in data .items ():
825
- assert typ in self . all_data , ('unpacked data contains '
826
- 'extra key "{0}"'
827
- .format (typ ))
840
+ assert typ in all_data , ('unpacked data contains '
841
+ 'extra key "{0}"'
842
+ .format (typ ))
828
843
for dt , result in dv .items ():
829
- assert dt in self . all_data [typ ], ('data["{0}"] contains extra '
830
- 'key "{1}"' .format (typ , dt ))
844
+ assert dt in current_data [typ ], ('data["{0}"] contains extra '
845
+ 'key "{1}"' .format (typ , dt ))
831
846
try :
832
- expected = self . data [typ ][dt ]
847
+ expected = current_data [typ ][dt ]
833
848
except KeyError :
834
849
continue
835
850
@@ -862,9 +877,11 @@ def compare_frame_dt_mixed_tzs(self, result, expected, typ, version):
862
877
else :
863
878
tm .assert_frame_equal (result , expected )
864
879
865
- def read_msgpacks (self , version ):
880
+ @pytest .mark .parametrize ('version' , legacy_packers_versions ())
881
+ def test_msgpacks_legacy (self , current_packers_data , all_packers_data ,
882
+ version ):
866
883
867
- pth = tm .get_data_path ('legacy_msgpack/{0}' .format (str ( version ) ))
884
+ pth = tm .get_data_path ('legacy_msgpack/{0}' .format (version ))
868
885
n = 0
869
886
for f in os .listdir (pth ):
870
887
# GH12142 0.17 files packed in P2 can't be read in P3
@@ -873,19 +890,10 @@ def read_msgpacks(self, version):
873
890
continue
874
891
vf = os .path .join (pth , f )
875
892
try :
876
- self .compare (vf , version )
893
+ self .compare (current_packers_data , all_packers_data ,
894
+ vf , version )
877
895
except ImportError :
878
896
# blosc not installed
879
897
continue
880
898
n += 1
881
899
assert n > 0 , 'Msgpack files are not tested'
882
-
883
- def test_msgpack (self ):
884
- msgpack_path = tm .get_data_path ('legacy_msgpack' )
885
- n = 0
886
- for v in os .listdir (msgpack_path ):
887
- pth = os .path .join (msgpack_path , v )
888
- if os .path .isdir (pth ):
889
- yield self .read_msgpacks , v
890
- n += 1
891
- assert n > 0 , 'Msgpack files are not tested'
0 commit comments