From 973a2d81738413afcd553dba5c9a461cd52f187b Mon Sep 17 00:00:00 2001 From: Licht-T Date: Fri, 3 Nov 2017 20:10:46 +0900 Subject: [PATCH 1/7] BUG: Fix the error when reading the compressed UTF-16 file --- pandas/_libs/parsers.pyx | 28 +++++++++++++++++----------- pandas/io/parsers.py | 3 ++- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx index a5ce6c560d844..e2d9cdd0540a4 100644 --- a/pandas/_libs/parsers.pyx +++ b/pandas/_libs/parsers.pyx @@ -374,6 +374,17 @@ cdef class TextReader: float_precision=None, skip_blank_lines=True): + # encoding + if encoding is not None: + if not isinstance(encoding, bytes): + encoding = encoding.encode('utf-8') + encoding = encoding.lower() + self.c_encoding = encoding + else: + self.c_encoding = NULL + + self.encoding = encoding + self.parser = parser_new() self.parser.chunksize = tokenize_chunksize @@ -495,17 +506,6 @@ cdef class TextReader: self.parser.double_converter_nogil = NULL self.parser.double_converter_withgil = round_trip - # encoding - if encoding is not None: - if not isinstance(encoding, bytes): - encoding = encoding.encode('utf-8') - encoding = encoding.lower() - self.c_encoding = encoding - else: - self.c_encoding = NULL - - self.encoding = encoding - if isinstance(dtype, dict): dtype = {k: pandas_dtype(dtype[k]) for k in dtype} @@ -684,6 +684,12 @@ cdef class TextReader: else: raise ValueError('Unrecognized compression type: %s' % self.compression) + + if b'utf-16' in (self.encoding or b''): + source = com.UTF8Recoder(source, self.encoding.decode('utf-8')) + self.encoding = b'utf-8' + self.c_encoding = self.encoding + self.handle = source if isinstance(source, basestring): diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 1b6414ea974fa..3108390af62d2 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -1671,7 +1671,8 @@ def __init__(self, src, **kwds): ParserBase.__init__(self, kwds) - if 'utf-16' in (kwds.get('encoding') or ''): + if kwds.get('compression') is None \ + and 'utf-16' in (kwds.get('encoding') or ''): if isinstance(src, compat.string_types): src = open(src, 'rb') self.handles.append(src) From 52d4266e3025508a6287358d4b865b23c03f9cf9 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Fri, 3 Nov 2017 20:39:48 +0900 Subject: [PATCH 2/7] TST: Add test for reading the zipped UTF-16 file --- pandas/tests/io/parser/common.py | 10 ++++++++++ pandas/tests/io/parser/data/utf16_ex.zip | Bin 0 -> 1680 bytes 2 files changed, 10 insertions(+) create mode 100644 pandas/tests/io/parser/data/utf16_ex.zip diff --git a/pandas/tests/io/parser/common.py b/pandas/tests/io/parser/common.py index 6a996213b28bb..3c29ca08320b4 100644 --- a/pandas/tests/io/parser/common.py +++ b/pandas/tests/io/parser/common.py @@ -750,6 +750,16 @@ def test_utf16_example(self): result = self.read_table(buf, encoding='utf-16') assert len(result) == 50 + def test_compressed_utf16_example(self): + # GH18071 + path = tm.get_data_path('utf16_ex.zip') + expected_path = tm.get_data_path('utf16_ex.txt') + + result = self.read_table(path, encoding='utf-16', compression='zip') + expected = self.read_table(expected_path, encoding='utf-16') + + tm.assert_frame_equal(result, expected) + def test_unicode_encoding(self): pth = tm.get_data_path('unicode_series.csv') diff --git a/pandas/tests/io/parser/data/utf16_ex.zip b/pandas/tests/io/parser/data/utf16_ex.zip new file mode 100644 index 0000000000000000000000000000000000000000..f5fb659f4e433c874b8e1110d6c0b9b67094767b GIT binary patch literal 1680 zcmV;B25*{8@E)3jhHG^#K3?1QY-O0Nq;6QsYJt?yFOkckmOiflcM-=mkTd7D539 zmAwfpjIu%|l5q;&ZXbo?9yp}-wudC$Uyo(U*64ROW>Z^Y<>~IPr+a#Orf2le-`g~y zdx~gEF+EU^PWjE_$ER1cO##12>lxlcZtc;Iq4${XfjeZ3F{4P^1#`aV=Z=4k_nJenyNd8nQAAyb1SOsUnj=!;~c!ngaY?w(^Yr1gF>$ z!;%3Vv1R-G4467(nInEX#M@>i#}<7{-_SXuMf^O%7a{ko$2_l?tH-^Mxh;W3K6=t) z-0ztBBR`vr(4${iZp=OH(+SI7^W(#|ZMxz28Ib$5%Shs)cQchZu_~=5h=zQ*J%u)|-^BtEYN<6+wBQ8u8j`&2+Za z(aw)Qr+$=O#wDV;Pw&uE&y3R){u4ZjH+P)t(b>jNukly?bOB7wL-SGX^E{TA8zZ8t zIkMTJJ)TXvt&N*2tD>L|u)_`D$GDUxza=lt$ScT-P2}3?Z*I3*JhyruJ%~(x*UXXd zZhK;APldFs-?R4G(aI#Mb=1KXKOC{uSK#pACC^wNE&a}WlXU?LuL-gW``_0$1x;=6swq$pS#WF z?G$%*4Kc!@S;^keyrLM9o;p;~^W)GBtBUT<>J!B7jmaE+vzm{v#=az)ON%AP-=M;q zFuK(2dS>`oNzLbkWzL`#XPHKKR-eI_vKqW6JqMNDn#`%h9Fnv!h*((}h~r2b&+Z&AN2SJeNSJm*XRxX-6gl4O#P@Io)|Q{;lH3=lt$? z)f{epunU=py&Rek-3EOAeI z{&S8P3C$^TaeMw$x|yu4-K|C08;(O%Jc6J#S@wownO1bNWXEVrE7I-OezC)cj(e^} zgG`0~v9L8;g*A~YZe2!jYnxQDpGog=^;{~!ZFd=ExeC6okRKqk=ILTnUHLr4X8$|j zXa(4IEk7U+RisGvO@-;1+oUJd9X>K;VqHz$O5~~D3f@LXgfdxIe%?>(^b@QzSM-Vh z+S~J|t;;-<+C0VhuB=+U^=)#?{E}~=n|dO9mvfAhRIOo)+xzsvoRf9q^S`jXcxT8l zBm0JtvAyhOcPqWueQk2r;tQE`WNzj3(|jfvGAcXZ8XG;!vRU5$p4QC^d*u+khFRV(%96~R^1OR?C{fS!^XWnYBvuYLE@IQ zHon0A^r50XTirC3Eov7FdIkSpTKm6HO928N0~7!N00;mxx(!QZ8zGzG1pok!EC2uu z02lxO000000001_fdBvi0CjX`F*aXicrJ8!bX8OZ00V+wkXdzj3jhHG^#K3?1QY-O a08mQ>1^@s60096208#(|04N3k0002W1w$nO literal 0 HcmV?d00001 From abfdaddadeb0ff73decd1c02e03489d123c6c79b Mon Sep 17 00:00:00 2001 From: Licht-T Date: Fri, 3 Nov 2017 22:21:39 +0900 Subject: [PATCH 3/7] DOC: Add comments about UTF-16 source conversion --- pandas/_libs/parsers.pyx | 2 ++ pandas/io/parsers.py | 1 + 2 files changed, 3 insertions(+) diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx index e2d9cdd0540a4..c6c1a148792bc 100644 --- a/pandas/_libs/parsers.pyx +++ b/pandas/_libs/parsers.pyx @@ -686,6 +686,8 @@ cdef class TextReader: self.compression) if b'utf-16' in (self.encoding or b''): + # we need to read utf-16 through UTF8Recoder. + # if source is utf-16, convert source to utf-8 by UTF8Recoder. source = com.UTF8Recoder(source, self.encoding.decode('utf-8')) self.encoding = b'utf-8' self.c_encoding = self.encoding diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index 3108390af62d2..d761e6bf01199 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -1673,6 +1673,7 @@ def __init__(self, src, **kwds): if kwds.get('compression') is None \ and 'utf-16' in (kwds.get('encoding') or ''): + # if source is utf-16 plain text, convert source to utf-8 if isinstance(src, compat.string_types): src = open(src, 'rb') self.handles.append(src) From bacf224d275fc53fbd535d6ad1d37bea711f35cc Mon Sep 17 00:00:00 2001 From: Licht-T Date: Fri, 3 Nov 2017 22:31:11 +0900 Subject: [PATCH 4/7] DOC: Add whatsnew note about fixing the bug of reading compressed UTF-16 file --- doc/source/whatsnew/v0.21.1.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.21.1.txt b/doc/source/whatsnew/v0.21.1.txt index 4adafe7c06450..270d81dda6b45 100644 --- a/doc/source/whatsnew/v0.21.1.txt +++ b/doc/source/whatsnew/v0.21.1.txt @@ -76,7 +76,7 @@ I/O ^^^ - Bug in class:`~pandas.io.stata.StataReader` not converting date/time columns with display formatting addressed (:issue:`17990`). Previously columns with display formatting were normally left as ordinal numbers and not converted to datetime objects. - +- Bug in :func:`read_csv` when reading a compressed UTF-16 encoded file (:issue:`18071`) Plotting ^^^^^^^^ From b2a3f972ad5d89a8e63daf793d3c44f7565f6946 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Sat, 4 Nov 2017 04:41:35 +0900 Subject: [PATCH 5/7] TST: Move and change the test case --- pandas/tests/io/parser/common.py | 10 ---------- pandas/tests/io/parser/compression.py | 14 ++++++++++++++ pandas/tests/io/parser/data/utf16_ex.zip | Bin 1680 -> 0 bytes pandas/tests/io/parser/data/utf16_ex_small.zip | Bin 0 -> 285 bytes 4 files changed, 14 insertions(+), 10 deletions(-) delete mode 100644 pandas/tests/io/parser/data/utf16_ex.zip create mode 100644 pandas/tests/io/parser/data/utf16_ex_small.zip diff --git a/pandas/tests/io/parser/common.py b/pandas/tests/io/parser/common.py index 3c29ca08320b4..6a996213b28bb 100644 --- a/pandas/tests/io/parser/common.py +++ b/pandas/tests/io/parser/common.py @@ -750,16 +750,6 @@ def test_utf16_example(self): result = self.read_table(buf, encoding='utf-16') assert len(result) == 50 - def test_compressed_utf16_example(self): - # GH18071 - path = tm.get_data_path('utf16_ex.zip') - expected_path = tm.get_data_path('utf16_ex.txt') - - result = self.read_table(path, encoding='utf-16', compression='zip') - expected = self.read_table(expected_path, encoding='utf-16') - - tm.assert_frame_equal(result, expected) - def test_unicode_encoding(self): pth = tm.get_data_path('unicode_series.csv') diff --git a/pandas/tests/io/parser/compression.py b/pandas/tests/io/parser/compression.py index 797c12139656d..84db9d14eee07 100644 --- a/pandas/tests/io/parser/compression.py +++ b/pandas/tests/io/parser/compression.py @@ -7,6 +7,7 @@ import pytest +import pandas as pd import pandas.util.testing as tm @@ -157,6 +158,19 @@ def test_read_csv_infer_compression(self): inputs[3].close() + def test_read_csv_compressed_utf16_example(self): + # GH18071 + path = tm.get_data_path('utf16_ex_small.zip') + + result = self.read_csv(path, encoding='utf-16', + compression='zip', sep='\t') + expected = pd.DataFrame({ + u'Country': [u'Venezuela', u'Venezuela'], + u'Twitter': [u'Hugo Chávez Frías', u'Henrique Capriles R.'] + }) + + tm.assert_frame_equal(result, expected) + def test_invalid_compression(self): msg = 'Unrecognized compression type: sfark' with tm.assert_raises_regex(ValueError, msg): diff --git a/pandas/tests/io/parser/data/utf16_ex.zip b/pandas/tests/io/parser/data/utf16_ex.zip deleted file mode 100644 index f5fb659f4e433c874b8e1110d6c0b9b67094767b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1680 zcmV;B25*{8@E)3jhHG^#K3?1QY-O0Nq;6QsYJt?yFOkckmOiflcM-=mkTd7D539 zmAwfpjIu%|l5q;&ZXbo?9yp}-wudC$Uyo(U*64ROW>Z^Y<>~IPr+a#Orf2le-`g~y zdx~gEF+EU^PWjE_$ER1cO##12>lxlcZtc;Iq4${XfjeZ3F{4P^1#`aV=Z=4k_nJenyNd8nQAAyb1SOsUnj=!;~c!ngaY?w(^Yr1gF>$ z!;%3Vv1R-G4467(nInEX#M@>i#}<7{-_SXuMf^O%7a{ko$2_l?tH-^Mxh;W3K6=t) z-0ztBBR`vr(4${iZp=OH(+SI7^W(#|ZMxz28Ib$5%Shs)cQchZu_~=5h=zQ*J%u)|-^BtEYN<6+wBQ8u8j`&2+Za z(aw)Qr+$=O#wDV;Pw&uE&y3R){u4ZjH+P)t(b>jNukly?bOB7wL-SGX^E{TA8zZ8t zIkMTJJ)TXvt&N*2tD>L|u)_`D$GDUxza=lt$ScT-P2}3?Z*I3*JhyruJ%~(x*UXXd zZhK;APldFs-?R4G(aI#Mb=1KXKOC{uSK#pACC^wNE&a}WlXU?LuL-gW``_0$1x;=6swq$pS#WF z?G$%*4Kc!@S;^keyrLM9o;p;~^W)GBtBUT<>J!B7jmaE+vzm{v#=az)ON%AP-=M;q zFuK(2dS>`oNzLbkWzL`#XPHKKR-eI_vKqW6JqMNDn#`%h9Fnv!h*((}h~r2b&+Z&AN2SJeNSJm*XRxX-6gl4O#P@Io)|Q{;lH3=lt$? z)f{epunU=py&Rek-3EOAeI z{&S8P3C$^TaeMw$x|yu4-K|C08;(O%Jc6J#S@wownO1bNWXEVrE7I-OezC)cj(e^} zgG`0~v9L8;g*A~YZe2!jYnxQDpGog=^;{~!ZFd=ExeC6okRKqk=ILTnUHLr4X8$|j zXa(4IEk7U+RisGvO@-;1+oUJd9X>K;VqHz$O5~~D3f@LXgfdxIe%?>(^b@QzSM-Vh z+S~J|t;;-<+C0VhuB=+U^=)#?{E}~=n|dO9mvfAhRIOo)+xzsvoRf9q^S`jXcxT8l zBm0JtvAyhOcPqWueQk2r;tQE`WNzj3(|jfvGAcXZ8XG;!vRU5$p4QC^d*u+khFRV(%96~R^1OR?C{fS!^XWnYBvuYLE@IQ zHon0A^r50XTirC3Eov7FdIkSpTKm6HO928N0~7!N00;mxx(!QZ8zGzG1pok!EC2uu z02lxO000000001_fdBvi0CjX`F*aXicrJ8!bX8OZ00V+wkXdzj3jhHG^#K3?1QY-O a08mQ>1^@s60096208#(|04N3k0002W1w$nO diff --git a/pandas/tests/io/parser/data/utf16_ex_small.zip b/pandas/tests/io/parser/data/utf16_ex_small.zip new file mode 100644 index 0000000000000000000000000000000000000000..b0560c1b1f6c41307b575f2a86509021b49649f4 GIT binary patch literal 285 zcmWIWW@Zs#U|`^2c&?n{%@J5Cmki|10Ae8q8HUo5G()ra)Qb4x+{Bz5y^@NO&=5`r zX2v6bB0;#cf}4SnIF&U?+Ut|9%K z*xYyP>aL!jYp@|<#>O`+A~_|u(y}H_2)et1Z)sMQR;tn2-aQ>H(-D~jm`P25a**gJ;0ll4Wxq+ M2qS>>dJu;J0ImR5j{pDw literal 0 HcmV?d00001 From 012b4968f84154971ec3118a7ecd8614d5d83efb Mon Sep 17 00:00:00 2001 From: Licht-T Date: Sat, 4 Nov 2017 05:05:59 +0900 Subject: [PATCH 6/7] Use parentheses instead of the backslash to wrap multi-line conditional --- pandas/io/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index d761e6bf01199..7f3f5630e49f9 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -1671,8 +1671,8 @@ def __init__(self, src, **kwds): ParserBase.__init__(self, kwds) - if kwds.get('compression') is None \ - and 'utf-16' in (kwds.get('encoding') or ''): + if (kwds.get('compression') is None + and 'utf-16' in (kwds.get('encoding') or '')): # if source is utf-16 plain text, convert source to utf-8 if isinstance(src, compat.string_types): src = open(src, 'rb') From 1a06857f6189d9c147ad4ff9f3b90f5ee8d51e89 Mon Sep 17 00:00:00 2001 From: Licht-T Date: Sat, 4 Nov 2017 05:28:48 +0900 Subject: [PATCH 7/7] Add comment for encoding settings --- pandas/_libs/parsers.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx index c6c1a148792bc..85857c158f96e 100644 --- a/pandas/_libs/parsers.pyx +++ b/pandas/_libs/parsers.pyx @@ -374,7 +374,7 @@ cdef class TextReader: float_precision=None, skip_blank_lines=True): - # encoding + # set encoding for native Python and C library if encoding is not None: if not isinstance(encoding, bytes): encoding = encoding.encode('utf-8')