From 66c7e8dd4614338e20192fc11b891f1d77276d37 Mon Sep 17 00:00:00 2001 From: faulaire Date: Mon, 31 Dec 2018 00:57:11 +0100 Subject: [PATCH 1/8] Fix ValueError when reading a Dataframe with HDFStore in Python 3 from fixed format written in Python 2 --- pandas/io/pytables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 5b76b4bb3d6ab..b3de0a997308a 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -2501,7 +2501,7 @@ def set_attrs(self): def get_attrs(self): """ retrieve our attributes """ self.encoding = _ensure_encoding(getattr(self.attrs, 'encoding', None)) - self.errors = getattr(self.attrs, 'errors', 'strict') + self.errors = _ensure_decoded(getattr(self.attrs, 'errors', 'strict')) for n in self.attributes: setattr(self, n, _ensure_decoded(getattr(self.attrs, n, None))) From 5fff78603dae55952eba63f5e7f8f0d9c187580b Mon Sep 17 00:00:00 2001 From: Fabien Date: Mon, 31 Dec 2018 17:45:35 +0100 Subject: [PATCH 2/8] ValueError when reading a Dataframe with HDFStore in Python 3 from fixed format written in Python 2 --- doc/source/whatsnew/v0.24.0.rst | 1 + pandas/io/pytables.py | 2 ++ .../data/legacy_hdf/legacy_table_fixed_py2.h5 | Bin 0 -> 1064200 bytes pandas/tests/io/test_pytables.py | 13 +++++++++++++ 4 files changed, 16 insertions(+) create mode 100644 pandas/tests/io/data/legacy_hdf/legacy_table_fixed_py2.h5 diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index b4331aab3085f..025731a70c2cd 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1515,6 +1515,7 @@ MultiIndex I/O ^^^ +- Fix ``ValueError`` when reading a Dataframe with HDFStore in Python 3 from fixed format written in Python 2 .. _whatsnew_0240.bug_fixes.nan_with_str_dtype: diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index b3de0a997308a..ef738a8b0ffe5 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -2661,6 +2661,8 @@ def read_index_node(self, node, start=None, stop=None): if 'name' in node._v_attrs: name = _ensure_str(node._v_attrs.name) + if isinstance(name, compat.binary_type): + name = _ensure_decoded(name) index_class = self._alias_to_class(_ensure_decoded( getattr(node._v_attrs, 'index_class', ''))) diff --git a/pandas/tests/io/data/legacy_hdf/legacy_table_fixed_py2.h5 b/pandas/tests/io/data/legacy_hdf/legacy_table_fixed_py2.h5 new file mode 100644 index 0000000000000000000000000000000000000000..540251d9fae86f0d691e0e7748e0249af55e1d54 GIT binary patch literal 1064200 zcmeI(&2HSr0RUjGWIJ0~t<|PMe-F77FnSOyqXB#gnvEq#u4Kv3kD&l9l$Eqe)mSTO zwMyfnhaCEoyg?tK$38-jKK9s0=rM25a{k2ax{~YGMvUa6AlST<`~L%yZ*OvcVoFOz}2huU)Xoqe~eVm#^c{!?yRrZ>qGp@^uhYFm;UDLTu%3D%yaxcoyj-c=Xr-uyH97X#_aOZq~GRW9f9%tw3O4MTz?md?#KHl zq2z`zl)#;bhr1u%*#4;d@U(Z*Ebk9{2VpU7pBvjgw|%yKePwmNRhQsgT&wn_kgB;) zZsg_rW5L-Tq$pWPAANs27iSyk32O z&^-uSrmK?f+Gx6udV`bw!+xrdLlG(E_Qveo8?!e@(|s+b>-BdJ_xAfA#GTc%u)pvq zD(O~c)0W2Vm6UEUI2_a^I>aB{&&hDGzdIb=&*gY#{k{EzkW;9KQ`vqiES~MBJpP*V z`bOMOyua$(Ww75HKC0t~{k<}hZqWPS^kH{U=W{LPl@GjJ|E+J=k?ZO66r9Z`>)b=s zkY5Nd=jDlsiI_t>xAZ1kThl2~zVXsR^{spsA@c8T|M=&PPbydEJr_G`Yn^xNG)?@A zpNltVe-igNUA-Q|^qju*#B;j#D}*s#sz<`HL+i@T{pb2aTLwcJX>qlC@=LW{LseYoF`DgRSDJhbY^aQs?;B0KJiNv}ymYp3(x$5P(%JO90&l%Q@zh|Cbj<$0-DsL?51eBwHzrMfB?`O^R zy%i(0s@GyYl>5D6Kh{XKaKZa}suv!gUFYOq>e=RcrsEi`7ebQrujjAT3z6)Ld47a` ztZ;qa%Gb5j5A9rj-iyc9%qHvSYls@EkI)AZ{!-F({AbsBGOa%AmzkLO`w#25acbqk zje6l(?%UCRqzB8XXQrvAFYcF_d&Yj5=J|Ow-Y@sp7dGctI~%L_(kYBTzYlkQ)eHS- zVVolL2bXcg5+EPMX2hM1TMR0t5(rIR)Oiy}BHq`1ygKp8oZ?Nj}|A zCU#?2#O&_jpf`JP`f#}49SpjUrnU#Y-NSx8e0RD#oH}l$SX1?2WZZf@u~hwjv$gZb zwd2X_ORezl4>`(S7^|E*zO*^9dTXg#YSmFLUtgNsZ2fj+W#wm~y%J|jKOOFT_KV{y zo2~ioa3_x{*5AQg`*L0fXGMSj0RjXF5FkK+009C7UZlWzGO3S4#ZnpYGfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly iK!5-N0t5&UAV7cs0RjXF5FkK+009C72oQL>z`p<-NVALp literal 0 HcmV?d00001 diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py index 1c4d00c8b3e15..db84f60920db4 100644 --- a/pandas/tests/io/test_pytables.py +++ b/pandas/tests/io/test_pytables.py @@ -4540,6 +4540,19 @@ def test_pytables_native2_read(self, datapath): d1 = store['detector'] assert isinstance(d1, DataFrame) + def test_legacy_table_fixed_format_read_py2(self, datapath): + # legacy table with fixed format written en Python 2 + with ensure_clean_store( + datapath('io', 'data', 'legacy_hdf', 'legacy_table_fixed_py2.h5'), + mode='r') as store: + with catch_warnings(): + simplefilter("ignore", pd.io.pytables.IncompatibilityWarning) + result = store.select('df') + expected = pd.DataFrame([[1, 2, 3, 'D']], + columns=['A', 'B', 'C', 'D'], + index=pd.Index(['ABC'], name='INDEX_NAME')) + assert_frame_equal(expected, result) + def test_legacy_table_read(self, datapath): # legacy table types with ensure_clean_store( From e80ea1bf9bb90f18f8a7f1b756241aaa8c3d0b9a Mon Sep 17 00:00:00 2001 From: Fabien Aulaire <306648+faulaire@users.noreply.github.com> Date: Mon, 31 Dec 2018 17:54:55 +0100 Subject: [PATCH 3/8] Update v0.24.0.rst Add issue number to whatsnew --- doc/source/whatsnew/v0.24.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 025731a70c2cd..11968d981d840 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1515,7 +1515,7 @@ MultiIndex I/O ^^^ -- Fix ``ValueError`` when reading a Dataframe with HDFStore in Python 3 from fixed format written in Python 2 +- Fix ``ValueError`` when reading a Dataframe with HDFStore in Python 3 from fixed format written in Python 2 (:issue:`24510`) .. _whatsnew_0240.bug_fixes.nan_with_str_dtype: From 0a7e2e43967fc1a1f7ca101e3f484181d94ef571 Mon Sep 17 00:00:00 2001 From: Fabien Aulaire <306648+faulaire@users.noreply.github.com> Date: Mon, 31 Dec 2018 17:57:26 +0100 Subject: [PATCH 4/8] Update pytables.py Try to decode index name systematically --- pandas/io/pytables.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index ef738a8b0ffe5..a894b8788f8d8 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -2661,8 +2661,7 @@ def read_index_node(self, node, start=None, stop=None): if 'name' in node._v_attrs: name = _ensure_str(node._v_attrs.name) - if isinstance(name, compat.binary_type): - name = _ensure_decoded(name) + name = _ensure_decoded(name) index_class = self._alias_to_class(_ensure_decoded( getattr(node._v_attrs, 'index_class', ''))) From 4f224f25179fc02725655b817b3d7562459d02a2 Mon Sep 17 00:00:00 2001 From: Fabien Aulaire <306648+faulaire@users.noreply.github.com> Date: Mon, 31 Dec 2018 18:00:36 +0100 Subject: [PATCH 5/8] Update test_pytables.py Remove useless simplefilter --- pandas/tests/io/test_pytables.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py index db84f60920db4..aca4570bf6d7b 100644 --- a/pandas/tests/io/test_pytables.py +++ b/pandas/tests/io/test_pytables.py @@ -4546,7 +4546,6 @@ def test_legacy_table_fixed_format_read_py2(self, datapath): datapath('io', 'data', 'legacy_hdf', 'legacy_table_fixed_py2.h5'), mode='r') as store: with catch_warnings(): - simplefilter("ignore", pd.io.pytables.IncompatibilityWarning) result = store.select('df') expected = pd.DataFrame([[1, 2, 3, 'D']], columns=['A', 'B', 'C', 'D'], From 554d0c52f895063ec33bc18dae578ad2cf0a8a5c Mon Sep 17 00:00:00 2001 From: faulaire Date: Mon, 31 Dec 2018 19:05:39 +0100 Subject: [PATCH 6/8] Fix flake8 error on test_pytables.py --- pandas/tests/io/test_pytables.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py index aca4570bf6d7b..da12a5dcdcf2c 100644 --- a/pandas/tests/io/test_pytables.py +++ b/pandas/tests/io/test_pytables.py @@ -4543,13 +4543,15 @@ def test_pytables_native2_read(self, datapath): def test_legacy_table_fixed_format_read_py2(self, datapath): # legacy table with fixed format written en Python 2 with ensure_clean_store( - datapath('io', 'data', 'legacy_hdf', 'legacy_table_fixed_py2.h5'), + datapath('io', 'data', 'legacy_hdf', + 'legacy_table_fixed_py2.h5'), mode='r') as store: with catch_warnings(): result = store.select('df') expected = pd.DataFrame([[1, 2, 3, 'D']], columns=['A', 'B', 'C', 'D'], - index=pd.Index(['ABC'], name='INDEX_NAME')) + index=pd.Index(['ABC'], + name='INDEX_NAME')) assert_frame_equal(expected, result) def test_legacy_table_read(self, datapath): From a33fb2cc01841ae37ae5fc9221963b70117fb097 Mon Sep 17 00:00:00 2001 From: faulaire Date: Mon, 31 Dec 2018 21:45:15 +0100 Subject: [PATCH 7/8] Add GH issue number as comment in tests Change whatsnew category --- doc/source/whatsnew/v0.24.0.rst | 3 +-- pandas/tests/io/test_pytables.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 11968d981d840..78667b0e3e39b 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1515,8 +1515,6 @@ MultiIndex I/O ^^^ -- Fix ``ValueError`` when reading a Dataframe with HDFStore in Python 3 from fixed format written in Python 2 (:issue:`24510`) - .. _whatsnew_0240.bug_fixes.nan_with_str_dtype: Proper handling of `np.NaN` in a string data-typed column with the Python engine @@ -1588,6 +1586,7 @@ Notice how we now instead output ``np.nan`` itself instead of a stringified form - Bug in :func:`pandas.io.json.json_normalize` that caused it to raise ``TypeError`` when two consecutive elements of ``record_path`` are dicts (:issue:`22706`) - Bug in :meth:`DataFrame.to_stata`, :class:`pandas.io.stata.StataWriter` and :class:`pandas.io.stata.StataWriter117` where a exception would leave a partially written and invalid dta file (:issue:`23573`) - Bug in :meth:`DataFrame.to_stata` and :class:`pandas.io.stata.StataWriter117` that produced invalid files when using strLs with non-ASCII characters (:issue:`23573`) +- Bug in :class:`HDFStore` that caused it to raise ``ValueError`` when reading a Dataframe in Python 3 from fixed format written in Python 2 (:issue:`24510`) Plotting ^^^^^^^^ diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py index da12a5dcdcf2c..f52d0040a7923 100644 --- a/pandas/tests/io/test_pytables.py +++ b/pandas/tests/io/test_pytables.py @@ -4541,6 +4541,7 @@ def test_pytables_native2_read(self, datapath): assert isinstance(d1, DataFrame) def test_legacy_table_fixed_format_read_py2(self, datapath): + # GH 24510 # legacy table with fixed format written en Python 2 with ensure_clean_store( datapath('io', 'data', 'legacy_hdf', From 173265ccf9bf2c80a3ff2b2e864c742eb04a8a88 Mon Sep 17 00:00:00 2001 From: faulaire Date: Tue, 1 Jan 2019 00:29:20 +0100 Subject: [PATCH 8/8] Remove useless catch_warnings context manager from test --- pandas/tests/io/test_pytables.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py index f52d0040a7923..4179e81d02042 100644 --- a/pandas/tests/io/test_pytables.py +++ b/pandas/tests/io/test_pytables.py @@ -4547,13 +4547,12 @@ def test_legacy_table_fixed_format_read_py2(self, datapath): datapath('io', 'data', 'legacy_hdf', 'legacy_table_fixed_py2.h5'), mode='r') as store: - with catch_warnings(): - result = store.select('df') - expected = pd.DataFrame([[1, 2, 3, 'D']], - columns=['A', 'B', 'C', 'D'], - index=pd.Index(['ABC'], - name='INDEX_NAME')) - assert_frame_equal(expected, result) + result = store.select('df') + expected = pd.DataFrame([[1, 2, 3, 'D']], + columns=['A', 'B', 'C', 'D'], + index=pd.Index(['ABC'], + name='INDEX_NAME')) + assert_frame_equal(expected, result) def test_legacy_table_read(self, datapath): # legacy table types