Skip to content

Commit a15ca97

Browse files
closes #44914 by changing the path object to string if it is of io.BufferWriter (#45480)
1 parent aeaca2f commit a15ca97

File tree

3 files changed

+26
-0
lines changed

3 files changed

+26
-0
lines changed

doc/source/whatsnew/v1.5.0.rst

+2
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,8 @@ I/O
233233
^^^
234234
- Bug in :meth:`DataFrame.to_stata` where no error is raised if the :class:`DataFrame` contains ``-np.inf`` (:issue:`45350`)
235235
- Bug in :meth:`DataFrame.info` where a new line at the end of the output is omitted when called on an empty :class:`DataFrame` (:issue:`45494`)
236+
- Bug in :func:`read_parquet` when ``engine="pyarrow"`` which caused partial write to disk when column of unsupported datatype was passed (:issue:`44914`)
237+
-
236238

237239
Period
238240
^^^^^^

pandas/io/parquet.py

+3
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,9 @@ def write(
180180
mode="wb",
181181
is_dir=partition_cols is not None,
182182
)
183+
if isinstance(path_or_handle, io.BufferedWriter):
184+
path_or_handle = path_or_handle.raw.name
185+
183186
try:
184187
if partition_cols is not None:
185188
# writes to multiple files under the given path

pandas/tests/io/test_parquet.py

+21
Original file line numberDiff line numberDiff line change
@@ -721,6 +721,27 @@ def test_unsupported(self, pa):
721721
# older pyarrows raise ArrowInvalid
722722
self.check_external_error_on_write(df, pa, pyarrow.ArrowException)
723723

724+
def test_unsupported_float16(self, pa):
725+
# #44847, #44914
726+
# Not able to write float 16 column using pyarrow.
727+
data = np.arange(2, 10, dtype=np.float16)
728+
df = pd.DataFrame(data=data, columns=["fp16"])
729+
self.check_external_error_on_write(df, pa, pyarrow.ArrowException)
730+
731+
@pytest.mark.parametrize("path_type", [str, pathlib.Path])
732+
def test_unsupported_float16_cleanup(self, pa, path_type):
733+
# #44847, #44914
734+
# Not able to write float 16 column using pyarrow.
735+
# Tests cleanup by pyarrow in case of an error
736+
data = np.arange(2, 10, dtype=np.float16)
737+
df = pd.DataFrame(data=data, columns=["fp16"])
738+
739+
with tm.ensure_clean() as path_str:
740+
path = path_type(path_str)
741+
with tm.external_error_raised(pyarrow.ArrowException):
742+
df.to_parquet(path=path, engine=pa)
743+
assert not os.path.isfile(path)
744+
724745
def test_categorical(self, pa):
725746

726747
# supported in >= 0.7.0

0 commit comments

Comments
 (0)