From 4d033f193980f071fe190309c662a535191c583b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Thu, 18 Aug 2022 20:10:26 -0400 Subject: [PATCH 1/5] TYP: fix ReadPickleBuffer --- pandas/_typing.py | 4 ++-- pandas/io/xml.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index af037ff0473e3..f717c694915e4 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -207,7 +207,7 @@ def tell(self) -> int: class ReadBuffer(BaseBuffer, Protocol[AnyStr_cov]): - def read(self, __n: int | None = ...) -> AnyStr_cov: + def read(self, __n: int = ...) -> AnyStr_cov: # for BytesIOWrapper, gzip.GzipFile, bz2.BZ2File ... @@ -223,7 +223,7 @@ def flush(self) -> Any: class ReadPickleBuffer(ReadBuffer[bytes], Protocol): - def readline(self) -> AnyStr_cov: + def readline(self) -> bytes: ... diff --git a/pandas/io/xml.py b/pandas/io/xml.py index 71d19b7861fc2..bea04078cf556 100644 --- a/pandas/io/xml.py +++ b/pandas/io/xml.py @@ -712,10 +712,7 @@ def get_data_from_filepath( storage_options=storage_options, ) as handle_obj: filepath_or_buffer = ( - # error: Incompatible types in assignment (expression has type - # "Union[str, IO[str]]", variable has type "Union[Union[str, - # PathLike[str]], bytes, ReadBuffer[bytes], ReadBuffer[str]]") - handle_obj.handle.read() # type: ignore[assignment] + handle_obj.handle.read() if hasattr(handle_obj.handle, "read") else handle_obj.handle ) From a463ee021b71a35dc3cd07e818d01998fd64f21a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 23 Sep 2022 22:07:20 -0400 Subject: [PATCH 2/5] require fileno only for ReadCsvBuffer --- .pre-commit-config.yaml | 2 +- pandas/_typing.py | 8 ++++---- pandas/io/common.py | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 966fda1ed710a..9cdffed3a02e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -98,7 +98,7 @@ repos: types: [python] stages: [manual] additional_dependencies: &pyright_dependencies - - pyright@1.1.264 + - pyright@1.1.272 - id: pyright_reportGeneralTypeIssues # note: assumes python env is setup and activated name: pyright reportGeneralTypeIssues diff --git a/pandas/_typing.py b/pandas/_typing.py index f717c694915e4..1ced8881e1425 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -188,10 +188,6 @@ def mode(self) -> str: # for _get_filepath_or_buffer ... - def fileno(self) -> int: - # for _MMapWrapper - ... - def seek(self, __offset: int, __whence: int = ...) -> int: # with one argument: gzip.GzipFile, bz2.BZ2File # with two arguments: zip.ZipFile, read_sas @@ -237,6 +233,10 @@ def __iter__(self) -> Iterator[AnyStr_cov]: # for engine=python ... + def fileno(self) -> int: + # for _MMapWrapper + ... + def readline(self) -> AnyStr_cov: # for engine=python ... diff --git a/pandas/io/common.py b/pandas/io/common.py index 2fae19df13f8b..31de2d735bd26 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -52,6 +52,7 @@ CompressionOptions, FilePath, ReadBuffer, + ReadCsvBuffer, StorageOptions, WriteBuffer, ) @@ -1109,6 +1110,9 @@ def _maybe_memory_map( if not memory_map: return handle, memory_map, handles + # mmap only used csv + handle = cast(ReadCsvBuffer, handle) + # need to open the file first if isinstance(handle, str): handle = open(handle, "rb") From 73ac940b6a8ee819dc0c30341653ff7866819deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 23 Sep 2022 22:12:39 -0400 Subject: [PATCH 3/5] fix comment --- pandas/io/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index 31de2d735bd26..367823f412f23 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -1110,7 +1110,7 @@ def _maybe_memory_map( if not memory_map: return handle, memory_map, handles - # mmap only used csv + # mmap used by only read_csv handle = cast(ReadCsvBuffer, handle) # need to open the file first From 6d628d20bc48d2e0e6dc986633c1f9eaaba8055a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Sat, 1 Oct 2022 10:13:13 -0400 Subject: [PATCH 4/5] bump pyright --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9cdffed3a02e4..2294d548857b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -98,7 +98,7 @@ repos: types: [python] stages: [manual] additional_dependencies: &pyright_dependencies - - pyright@1.1.272 + - pyright@1.1.273 - id: pyright_reportGeneralTypeIssues # note: assumes python env is setup and activated name: pyright reportGeneralTypeIssues From de10b24bf8623ebd343ac06a409eb6c2423ddcb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Mon, 24 Oct 2022 18:03:32 -0400 Subject: [PATCH 5/5] pyright --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18d2571b268cc..8ff7526b87521 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -102,7 +102,7 @@ repos: types: [python] stages: [manual] additional_dependencies: &pyright_dependencies - - pyright@1.1.273 + - pyright@1.1.276 - id: pyright_reportGeneralTypeIssues # note: assumes python env is setup and activated name: pyright reportGeneralTypeIssues