Skip to content

Commit 2762c4c

Browse files
committed
REGR: be able to read Stata files without reading them fully into memory
Fixes pandas-dev#48700 Regressed in pandas-dev#9245 Regressed in 2f0ada3
1 parent e25aa9d commit 2762c4c

File tree

2 files changed

+15
-6
lines changed

2 files changed

+15
-6
lines changed

doc/source/whatsnew/v1.6.0.rst

+2-1
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,8 @@ MultiIndex
229229
I/O
230230
^^^
231231
- Bug in :func:`read_sas` caused fragmentation of :class:`DataFrame` and raised :class:`.errors.PerformanceWarning` (:issue:`48595`)
232-
-
232+
- Regression in :class:`StataReader` caused all files to needlessly be buffered in memory (:issue:`48922`)
233+
233234

234235
Period
235236
^^^^^^

pandas/io/stata.py

+13-5
Original file line numberDiff line numberDiff line change
@@ -1164,15 +1164,23 @@ def __init__(
11641164
self._lines_read = 0
11651165

11661166
self._native_byteorder = _set_endianness(sys.byteorder)
1167-
with get_handle(
1167+
1168+
handles = get_handle(
11681169
path_or_buf,
11691170
"rb",
11701171
storage_options=storage_options,
11711172
is_text=False,
11721173
compression=compression,
1173-
) as handles:
1174-
# Copy to BytesIO, and ensure no encoding
1175-
self.path_or_buf = BytesIO(handles.handle.read())
1174+
)
1175+
if hasattr(handles.handle, "seekable") and handles.handle.seekable():
1176+
# If the handle is directly seekable, use it without an extra copy.
1177+
self.path_or_buf = handles.handle
1178+
self._close_file = handles.close
1179+
else:
1180+
# Copy to memory, and ensure no encoding.
1181+
with handles:
1182+
self.path_or_buf = BytesIO(handles.handle.read())
1183+
self._close_file = self.path_or_buf.close
11761184

11771185
self._read_header()
11781186
self._setup_dtype()
@@ -1192,7 +1200,7 @@ def __exit__(
11921200

11931201
def close(self) -> None:
11941202
"""close the handle if its open"""
1195-
self.path_or_buf.close()
1203+
self._close_file()
11961204

11971205
def _set_encoding(self) -> None:
11981206
"""

0 commit comments

Comments
 (0)