Skip to content

CLN: memory_map do not silently fail #44777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 6, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions doc/source/whatsnew/v1.4.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,8 @@ I/O
- Bug in :func:`read_csv` raising ``AttributeError`` when attempting to read a .csv file and infer index column dtype from an nullable integer type (:issue:`44079`)
- :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` with ``compression`` set to ``'zip'`` no longer create a zip file containing a file ending with ".zip". Instead, they try to infer the inner file name more smartly. (:issue:`39465`)
- Bug in :func:`read_csv` when passing simultaneously a parser in ``date_parser`` and ``parse_dates=False``, the parsing was still called (:issue:`44366`)
- Bug in :func:`read_csv` silently ignoring errors when failling to create a memory-mapped file (:issue:`44766`)
-

Period
^^^^^^
Expand Down
29 changes: 9 additions & 20 deletions pandas/io/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -1040,26 +1040,15 @@ def _maybe_memory_map(
handle = open(handle, mode)
handles.append(handle)

try:
# error: Argument 1 to "_MMapWrapper" has incompatible type "Union[IO[Any],
# RawIOBase, BufferedIOBase, TextIOBase, mmap]"; expected "IO[Any]"
wrapped = cast(
BaseBuffer,
_MMapWrapper(handle, encoding, errors, decode), # type: ignore[arg-type]
)
# error: "BaseBuffer" has no attribute "close"
handle.close() # type: ignore[attr-defined]
handles.remove(handle)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If a user opened a file, this would cause an error (it is not in handles).

handles.append(wrapped)
handle = wrapped
except Exception:
# we catch any errors that may have occurred
# because that is consistent with the lower-level
# functionality of the C engine (pd.read_csv), so
# leave the file handler as is then
memory_map = False

return handle, memory_map, handles
# error: Argument 1 to "_MMapWrapper" has incompatible type "Union[IO[Any],
# RawIOBase, BufferedIOBase, TextIOBase, mmap]"; expected "IO[Any]"
wrapped = cast(
BaseBuffer,
_MMapWrapper(handle, encoding, errors, decode), # type: ignore[arg-type]
)
handles.append(wrapped)

return wrapped, memory_map, handles


def file_exists(filepath_or_buffer: FilePath | BaseBuffer) -> bool:
Expand Down
19 changes: 0 additions & 19 deletions pandas/tests/io/parser/common/test_file_buffer_url.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,25 +347,6 @@ def test_read_csv_file_handle(all_parsers, io_class, encoding):
assert not handle.closed


def test_memory_map_file_handle_silent_fallback(all_parsers, compression):
"""
Do not fail for buffers with memory_map=True (cannot memory map BytesIO).
GH 37621
"""
parser = all_parsers
expected = DataFrame({"a": [1], "b": [2]})

handle = BytesIO()
expected.to_csv(handle, index=False, compression=compression, mode="wb")
handle.seek(0)

tm.assert_frame_equal(
parser.read_csv(handle, memory_map=True, compression=compression),
expected,
)


def test_memory_map_compression(all_parsers, compression):
"""
Support memory map for compressed files.
Expand Down
18 changes: 9 additions & 9 deletions pandas/tests/io/parser/test_read_fwf.py
Original file line number Diff line number Diff line change
Expand Up @@ -699,15 +699,15 @@ def test_encoding_mmap(memory_map):
GH 23254.
"""
encoding = "iso8859_1"
data = BytesIO(" 1 A Ä 2\n".encode(encoding))
df = read_fwf(
data,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This never tested memory_map since it silently failed.

header=None,
widths=[2, 2, 2, 2],
encoding=encoding,
memory_map=memory_map,
)
data.seek(0)
with tm.ensure_clean() as path:
Path(path).write_bytes(" 1 A Ä 2\n".encode(encoding))
df = read_fwf(
path,
header=None,
widths=[2, 2, 2, 2],
encoding=encoding,
memory_map=memory_map,
)
df_reference = DataFrame([[1, "A", "Ä", 2]])
tm.assert_frame_equal(df, df_reference)

Expand Down
6 changes: 6 additions & 0 deletions pandas/tests/io/test_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from io import (
BytesIO,
StringIO,
UnsupportedOperation,
)
import mmap
import os
Expand Down Expand Up @@ -602,3 +603,8 @@ def test_errno_attribute():
with pytest.raises(FileNotFoundError, match="\\[Errno 2\\]") as err:
pd.read_csv("doesnt_exist")
assert err.errno == errno.ENOENT


def test_fail_mmap():
with pytest.raises(UnsupportedOperation, match="fileno"):
icom.get_handle(BytesIO(), "rb", memory_map=True)