Skip to content

Commit abfdadd

Browse files
committed
DOC: Add comments about UTF-16 source conversion
1 parent 52d4266 commit abfdadd

File tree

2 files changed

+3
-0
lines changed

2 files changed

+3
-0
lines changed

pandas/_libs/parsers.pyx

+2
Original file line numberDiff line numberDiff line change
@@ -686,6 +686,8 @@ cdef class TextReader:
686686
self.compression)
687687

688688
if b'utf-16' in (self.encoding or b''):
689+
# we need to read utf-16 through UTF8Recoder.
690+
# if source is utf-16, convert source to utf-8 by UTF8Recoder.
689691
source = com.UTF8Recoder(source, self.encoding.decode('utf-8'))
690692
self.encoding = b'utf-8'
691693
self.c_encoding = <char*> self.encoding

pandas/io/parsers.py

+1
Original file line numberDiff line numberDiff line change
@@ -1673,6 +1673,7 @@ def __init__(self, src, **kwds):
16731673

16741674
if kwds.get('compression') is None \
16751675
and 'utf-16' in (kwds.get('encoding') or ''):
1676+
# if source is utf-16 plain text, convert source to utf-8
16761677
if isinstance(src, compat.string_types):
16771678
src = open(src, 'rb')
16781679
self.handles.append(src)

0 commit comments

Comments
 (0)