From 31af5a281af99454733d7a697c9e02a0e7f06195 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sat, 8 Apr 2023 23:11:20 -0400 Subject: [PATCH 01/24] Implementing initial logic to add engine_kwargs to DataFrame.to_excel --- pandas/core/generic.py | 6 ++++++ pandas/io/formats/excel.py | 11 ++++++++++- pandas/tests/io/excel/test_openpyxl.py | 1 - pandas/tests/io/excel/test_writers.py | 13 +++++++++++++ pandas/tests/io/excel/test_xlsxwriter.py | 2 +- 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 38bac75bd48b9..a739f857e421c 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2142,6 +2142,7 @@ def to_excel( inf_rep: str = "inf", freeze_panes: tuple[int, int] | None = None, storage_options: StorageOptions = None, + engine_kwargs: dict | None = None, ) -> None: """ Write {klass} to an Excel sheet. @@ -2198,6 +2199,8 @@ def to_excel( {storage_options} .. versionadded:: {storage_options_versionadded} + engine_kwargs: dict, optional + Arbitrary keyword arguments passed to excel engine. See Also -------- @@ -2250,6 +2253,8 @@ def to_excel( >>> df1.to_excel('output1.xlsx', engine='xlsxwriter') # doctest: +SKIP """ + if engine_kwargs is None: + engine_kwargs = {} df = self if isinstance(self, ABCDataFrame) else self.to_frame() @@ -2274,6 +2279,7 @@ def to_excel( freeze_panes=freeze_panes, engine=engine, storage_options=storage_options, + engine_kwargs=engine_kwargs, ) @final diff --git a/pandas/io/formats/excel.py b/pandas/io/formats/excel.py index da35716a5b239..3ce6a46cb42d5 100644 --- a/pandas/io/formats/excel.py +++ b/pandas/io/formats/excel.py @@ -901,6 +901,7 @@ def write( freeze_panes: tuple[int, int] | None = None, engine: str | None = None, storage_options: StorageOptions = None, + engine_kwargs: dict | None = None, ) -> None: """ writer : path-like, file-like, or ExcelWriter object @@ -922,6 +923,8 @@ def write( {storage_options} .. versionadded:: 1.2.0 + engine_kwargs: dict, optional + Arbitrary keyword arguments passed to excel engine. """ from pandas.io.excel import ExcelWriter @@ -932,6 +935,9 @@ def write( f"Max sheet size is: {self.max_rows}, {self.max_cols}" ) + if engine_kwargs is None: + engine_kwargs = {} + formatted_cells = self.get_formatted_cells() if isinstance(writer, ExcelWriter): need_save = False @@ -939,7 +945,10 @@ def write( # error: Cannot instantiate abstract class 'ExcelWriter' with abstract # attributes 'engine', 'save', 'supported_extensions' and 'write_cells' writer = ExcelWriter( # type: ignore[abstract] - writer, engine=engine, storage_options=storage_options + writer, + engine=engine, + storage_options=storage_options, + engine_kwargs=engine_kwargs ) need_save = True diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index d28296ec2e380..ad1f35fb7cf7b 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -96,7 +96,6 @@ def test_engine_kwargs_write(ext, iso_dates): # ExcelWriter won't allow us to close without writing something DataFrame().to_excel(writer) - def test_engine_kwargs_append_invalid(ext): # GH 43445 # test whether an invalid engine kwargs actually raises diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 9a8e4eff5470a..605de6804b7d8 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1115,6 +1115,19 @@ def test_bytes_io(self, engine): reread_df = pd.read_excel(bio, index_col=0) tm.assert_frame_equal(df, reread_df) + def test_engine_kwargs(self, engine, path): + df = DataFrame([{"A": 1, "B": 2}, {"A": 3, "B": 4}]) + msgs = { + "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword argument 'foo'", + "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", + "xlsxwriter": r"Workbook.__init__() got an unexpected keyword argument 'foo'", + } + + df.to_excel(path, + engine_kwargs={"foo": "bar"} + ) + + def test_write_lists_dict(self, path): # see gh-8188. df = DataFrame( diff --git a/pandas/tests/io/excel/test_xlsxwriter.py b/pandas/tests/io/excel/test_xlsxwriter.py index c4d02d71390cc..6f720de85a32f 100644 --- a/pandas/tests/io/excel/test_xlsxwriter.py +++ b/pandas/tests/io/excel/test_xlsxwriter.py @@ -1,4 +1,5 @@ import contextlib +import re import pytest @@ -68,7 +69,6 @@ def test_engine_kwargs(ext, nan_inf_to_errors): with ExcelWriter(f, engine="xlsxwriter", engine_kwargs=engine_kwargs) as writer: assert writer.book.nan_inf_to_errors == nan_inf_to_errors - def test_book_and_sheets_consistent(ext): # GH#45687 - Ensure sheets is updated if user modifies book with tm.ensure_clean(ext) as f: From 5943856e5a98b6a59e7dcdf277f71e8ee7e9896d Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sat, 13 May 2023 17:37:01 -0400 Subject: [PATCH 02/24] Implementing logic to add engine_kwards to to_excel. Adding unit tests --- pandas/io/excel/_xlsxwriter.py | 6 +++++- pandas/tests/io/excel/test_writers.py | 12 +++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pandas/io/excel/_xlsxwriter.py b/pandas/io/excel/_xlsxwriter.py index 8cb88403b2f87..e3162bf299edf 100644 --- a/pandas/io/excel/_xlsxwriter.py +++ b/pandas/io/excel/_xlsxwriter.py @@ -212,7 +212,11 @@ def __init__( engine_kwargs=engine_kwargs, ) - self._book = Workbook(self._handles.handle, **engine_kwargs) + try: + self._book = Workbook(self._handles.handle, **engine_kwargs) + except: + self._handles.handle.close() + raise @property def book(self): diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 605de6804b7d8..269c2e6c6e90d 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1117,16 +1117,22 @@ def test_bytes_io(self, engine): def test_engine_kwargs(self, engine, path): df = DataFrame([{"A": 1, "B": 2}, {"A": 3, "B": 4}]) + msgs = { "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword argument 'foo'", "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", "xlsxwriter": r"Workbook.__init__() got an unexpected keyword argument 'foo'", } - df.to_excel(path, - engine_kwargs={"foo": "bar"} - ) + # Handle change in error message for openpyxl (write and append mode) + if engine == "openpyxl" and os.path.exists(path): + msgs["openpyxl"] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" + else: + with pytest.raises(TypeError, match=re.escape(msgs[engine])): + df.to_excel(path, + engine_kwargs={"foo": "bar"}, + ) def test_write_lists_dict(self, path): # see gh-8188. From 4a7994f7706afcc95af9349e7f86fabc090dce2f Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sat, 13 May 2023 20:31:17 -0400 Subject: [PATCH 03/24] Documenting the enhancement for this GH issue --- doc/source/user_guide/io.rst | 9 +++++++++ doc/source/whatsnew/v2.1.0.rst | 1 + pandas/tests/io/excel/test_writers.py | 17 +++++++++++------ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst index c33d4ab92d4c6..a6945f94ca942 100644 --- a/doc/source/user_guide/io.rst +++ b/doc/source/user_guide/io.rst @@ -3761,6 +3761,15 @@ one can pass an :class:`~pandas.io.excel.ExcelWriter`. .. _io.excel_writing_buffer: +When using the ``engine_kwargs`` parameter, pandas will pass these arguments to the +engine.For this, it is important to know which function pandas is using internally. + +* For the engine openpyxl, pandas is using :func:`openpyxl.Workbook` to create a new sheet and :func:`openpyxl.load_workbook` to append data to an existing sheet. The openpyxl engine writes to (``.xlsx``) and (``.xlsm``) files. + +* For the engine xlsxwriter, pandas is using :func:`xlsxwriter.Workbook` to write to (``.xlsx``) files. + +* For the engine odf, pandas is using :func:`odf.opendocument.OpenDocumentSpreadsheet` to write to (``.ods``) files. + Writing Excel files to memory +++++++++++++++++++++++++++++ diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 107b22953ff79..a3d95c4a489ad 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -87,6 +87,7 @@ Other enhancements - :meth:`DataFrame.applymap` now uses the :meth:`~api.extensions.ExtensionArray.map` method of underlying :class:`api.extensions.ExtensionArray` instances (:issue:`52219`) - :meth:`arrays.SparseArray.map` now supports ``na_action`` (:issue:`52096`). - Add dtype of categories to ``repr`` information of :class:`CategoricalDtype` (:issue:`52179`) +- Adding ``engine_kwargs`` parameter to :meth:`DataFrame.to_excel` (:issue:`52368`) - .. --------------------------------------------------------------------------- diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 269c2e6c6e90d..1b54d0d0a13fb 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1119,20 +1119,25 @@ def test_engine_kwargs(self, engine, path): df = DataFrame([{"A": 1, "B": 2}, {"A": 3, "B": 4}]) msgs = { - "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword argument 'foo'", + "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword " + r"argument 'foo'", "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", - "xlsxwriter": r"Workbook.__init__() got an unexpected keyword argument 'foo'", + "xlsxwriter": r"Workbook.__init__() got an unexpected keyword " + r"argument 'foo'", } # Handle change in error message for openpyxl (write and append mode) if engine == "openpyxl" and os.path.exists(path): - msgs["openpyxl"] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" + msgs[ + "openpyxl" + ] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" else: with pytest.raises(TypeError, match=re.escape(msgs[engine])): - df.to_excel(path, - engine_kwargs={"foo": "bar"}, - ) + df.to_excel( + path, + engine_kwargs={"foo": "bar"}, + ) def test_write_lists_dict(self, path): # see gh-8188. From 72c5f80addf89b2e91959d073dd5eb9c99497479 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sat, 13 May 2023 21:22:41 -0400 Subject: [PATCH 04/24] Fixing formatting errors --- doc/source/whatsnew/v2.1.0.rst | 2 +- pandas/io/formats/excel.py | 2 +- pandas/tests/io/excel/test_xlsxwriter.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 8fc2babc902fc..e369716e30b13 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -97,9 +97,9 @@ Other enhancements - Let :meth:`DataFrame.to_feather` accept a non-default :class:`Index` and non-string column names (:issue:`51787`) - Performance improvement in :func:`read_csv` (:issue:`52632`) with ``engine="c"`` - :meth:`Categorical.from_codes` has gotten a ``validate`` parameter (:issue:`50975`) +- Adding ``engine_kwargs`` parameter to :meth:`DataFrame.to_excel` (:issue:`53220`) - Performance improvement in :func:`concat` with homogeneous ``np.float64`` or ``np.float32`` dtypes (:issue:`52685`) - Performance improvement in :meth:`DataFrame.filter` when ``items`` is given (:issue:`52941`) -- Adding ``engine_kwargs`` parameter to :meth:`DataFrame.to_excel` (:issue:`53220`) - .. --------------------------------------------------------------------------- diff --git a/pandas/io/formats/excel.py b/pandas/io/formats/excel.py index ed253183aed1c..41e5c22447833 100644 --- a/pandas/io/formats/excel.py +++ b/pandas/io/formats/excel.py @@ -948,7 +948,7 @@ def write( writer, engine=engine, storage_options=storage_options, - engine_kwargs=engine_kwargs + engine_kwargs=engine_kwargs, ) need_save = True diff --git a/pandas/tests/io/excel/test_xlsxwriter.py b/pandas/tests/io/excel/test_xlsxwriter.py index 6f720de85a32f..c4d02d71390cc 100644 --- a/pandas/tests/io/excel/test_xlsxwriter.py +++ b/pandas/tests/io/excel/test_xlsxwriter.py @@ -1,5 +1,4 @@ import contextlib -import re import pytest @@ -69,6 +68,7 @@ def test_engine_kwargs(ext, nan_inf_to_errors): with ExcelWriter(f, engine="xlsxwriter", engine_kwargs=engine_kwargs) as writer: assert writer.book.nan_inf_to_errors == nan_inf_to_errors + def test_book_and_sheets_consistent(ext): # GH#45687 - Ensure sheets is updated if user modifies book with tm.ensure_clean(ext) as f: From 5ad0e10d897d6a7ce1ca7a2ece2435cab5a1ec71 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sat, 13 May 2023 21:45:30 -0400 Subject: [PATCH 05/24] Fixing formatting issues --- pandas/tests/io/excel/test_openpyxl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index c3582589ec1da..b8d41164792e0 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -96,6 +96,7 @@ def test_engine_kwargs_write(ext, iso_dates): # ExcelWriter won't allow us to close without writing something DataFrame().to_excel(writer) + def test_engine_kwargs_append_invalid(ext): # GH 43445 # test whether an invalid engine kwargs actually raises From a1799a108142022094a2082209a5076a2dab9b81 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sun, 14 May 2023 13:52:10 -0400 Subject: [PATCH 06/24] Fixing documentation errors and fixing unit test errors. --- pandas/core/generic.py | 2 +- pandas/tests/io/excel/test_writers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 4820837d33571..e21e89084fbad 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2212,7 +2212,7 @@ def to_excel( {storage_options} .. versionadded:: {storage_options_versionadded} - engine_kwargs: dict, optional + engine_kwargs : dict Arbitrary keyword arguments passed to excel engine. See Also diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 1b54d0d0a13fb..9ef40446f731f 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1122,7 +1122,7 @@ def test_engine_kwargs(self, engine, path): "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword " r"argument 'foo'", "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", - "xlsxwriter": r"Workbook.__init__() got an unexpected keyword " + "xlsxwriter": r"__init__() got an unexpected keyword " r"argument 'foo'", } From 1d6677ff925127b56901db6e1da38f9214531f14 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Sun, 14 May 2023 13:56:41 -0400 Subject: [PATCH 07/24] Fixing unit test errors --- pandas/tests/io/excel/test_writers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 9ef40446f731f..02f496e4a78e9 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1122,8 +1122,7 @@ def test_engine_kwargs(self, engine, path): "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword " r"argument 'foo'", "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", - "xlsxwriter": r"__init__() got an unexpected keyword " - r"argument 'foo'", + "xlsxwriter": r"__init__() got an unexpected keyword argument 'foo'", } # Handle change in error message for openpyxl (write and append mode) From cca1f6c572dd46732bcf1446ec9911851a666932 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Mon, 15 May 2023 18:48:21 -0400 Subject: [PATCH 08/24] Updating discrepancies in documentation, restricting except statement to check for TypeError --- doc/source/user_guide/io.rst | 2 +- doc/source/whatsnew/v2.1.0.rst | 5 ++--- pandas/io/excel/_xlsxwriter.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst index de8eb9fe35098..90a8bd868b60b 100644 --- a/doc/source/user_guide/io.rst +++ b/doc/source/user_guide/io.rst @@ -3786,7 +3786,7 @@ one can pass an :class:`~pandas.io.excel.ExcelWriter`. .. _io.excel_writing_buffer: When using the ``engine_kwargs`` parameter, pandas will pass these arguments to the -engine.For this, it is important to know which function pandas is using internally. +engine. For this, it is important to know which function pandas is using internally. * For the engine openpyxl, pandas is using :func:`openpyxl.Workbook` to create a new sheet and :func:`openpyxl.load_workbook` to append data to an existing sheet. The openpyxl engine writes to (``.xlsx``) and (``.xlsm``) files. diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index e369716e30b13..e10230cc968e0 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -83,7 +83,7 @@ Other enhancements - :meth:`arrays.DatetimeArray.map`, :meth:`arrays.TimedeltaArray.map` and :meth:`arrays.PeriodArray.map` can now take a ``na_action`` argument (:issue:`51644`) - :meth:`arrays.SparseArray.map` now supports ``na_action`` (:issue:`52096`). - Add :meth:`diff()` and :meth:`round()` for :class:`Index` (:issue:`19708`) -- Add dtype of categories to ``repr`` information of :class:`CategoricalDtype` (:issue:`52179`)l +- Add dtype of categories to ``repr`` information of :class:`CategoricalDtype` (:issue:`52179`) - Added to the escape mode "latex-math" preserving without escaping all characters between "\(" and "\)" in formatter (:issue:`51903`) - Adding ``engine_kwargs`` parameter to :meth:`DataFrame.read_excel` (:issue:`52214`) - Classes that are useful for type-hinting have been added to the public API in the new submodule ``pandas.api.typing`` (:issue:`48577`) @@ -97,11 +97,10 @@ Other enhancements - Let :meth:`DataFrame.to_feather` accept a non-default :class:`Index` and non-string column names (:issue:`51787`) - Performance improvement in :func:`read_csv` (:issue:`52632`) with ``engine="c"`` - :meth:`Categorical.from_codes` has gotten a ``validate`` parameter (:issue:`50975`) -- Adding ``engine_kwargs`` parameter to :meth:`DataFrame.to_excel` (:issue:`53220`) +- Added ``engine_kwargs`` parameter to :meth:`DataFrame.to_excel` (:issue:`53220`) - Performance improvement in :func:`concat` with homogeneous ``np.float64`` or ``np.float32`` dtypes (:issue:`52685`) - Performance improvement in :meth:`DataFrame.filter` when ``items`` is given (:issue:`52941`) - - .. --------------------------------------------------------------------------- .. _whatsnew_210.notable_bug_fixes: diff --git a/pandas/io/excel/_xlsxwriter.py b/pandas/io/excel/_xlsxwriter.py index e3162bf299edf..d7262c2f62d94 100644 --- a/pandas/io/excel/_xlsxwriter.py +++ b/pandas/io/excel/_xlsxwriter.py @@ -214,7 +214,7 @@ def __init__( try: self._book = Workbook(self._handles.handle, **engine_kwargs) - except: + except TypeError: self._handles.handle.close() raise From 7d42a15f442e9b5778e92e171e3d1e076459060c Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Mon, 15 May 2023 20:49:17 -0400 Subject: [PATCH 09/24] Adding blank line after my contribution in the whatsnew file --- doc/source/whatsnew/v2.1.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index e10230cc968e0..316934c44f3ed 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -101,6 +101,7 @@ Other enhancements - Performance improvement in :func:`concat` with homogeneous ``np.float64`` or ``np.float32`` dtypes (:issue:`52685`) - Performance improvement in :meth:`DataFrame.filter` when ``items`` is given (:issue:`52941`) - + .. --------------------------------------------------------------------------- .. _whatsnew_210.notable_bug_fixes: From 0e86804a0b1648e92d6f9bd45df156a65de39116 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Mon, 15 May 2023 21:08:46 -0400 Subject: [PATCH 10/24] Fixing a discrepancy with my unit tests --- pandas/tests/io/excel/test_writers.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 02f496e4a78e9..04ac613619251 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1131,12 +1131,11 @@ def test_engine_kwargs(self, engine, path): "openpyxl" ] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" - else: - with pytest.raises(TypeError, match=re.escape(msgs[engine])): - df.to_excel( - path, - engine_kwargs={"foo": "bar"}, - ) + with pytest.raises(TypeError, match=re.escape(msgs[engine])): + df.to_excel( + path, + engine_kwargs={"foo": "bar"}, + ) def test_write_lists_dict(self, path): # see gh-8188. From b016edde18b93f256e2e9d37932a4858c99aaa9f Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Mon, 15 May 2023 21:39:45 -0400 Subject: [PATCH 11/24] Fixing discrepancy with unit test --- pandas/tests/io/excel/test_writers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 04ac613619251..628408d9b9c06 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1122,7 +1122,8 @@ def test_engine_kwargs(self, engine, path): "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword " r"argument 'foo'", "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", - "xlsxwriter": r"__init__() got an unexpected keyword argument 'foo'", + "xlsxwriter": r"Workbook.__init__() got an unexpected keyword " + r"argument 'foo'", } # Handle change in error message for openpyxl (write and append mode) From 7c63fc936ad6ec7fdb75b064e50231744485a0af Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Mon, 15 May 2023 22:15:22 -0400 Subject: [PATCH 12/24] Fixing discrepancy with unit test --- pandas/tests/io/excel/test_writers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 628408d9b9c06..e565b193891ed 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1124,6 +1124,7 @@ def test_engine_kwargs(self, engine, path): "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", "xlsxwriter": r"Workbook.__init__() got an unexpected keyword " r"argument 'foo'", + None: "Workbook.__init__() got an unexpected keyword argument 'foo'", } # Handle change in error message for openpyxl (write and append mode) From 954a72721c6a64ae94ed70eeb0515c1c5bc46e1a Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Mon, 15 May 2023 22:44:31 -0400 Subject: [PATCH 13/24] Fixing discrepancy with unit test --- pandas/tests/io/excel/test_writers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index e565b193891ed..245419f542985 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1124,7 +1124,7 @@ def test_engine_kwargs(self, engine, path): "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", "xlsxwriter": r"Workbook.__init__() got an unexpected keyword " r"argument 'foo'", - None: "Workbook.__init__() got an unexpected keyword argument 'foo'", + None: r"__init__() got an unexpected keyword argument 'foo'", } # Handle change in error message for openpyxl (write and append mode) From 9938978ceabcf334484fa780995b1da680ed5795 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Mon, 15 May 2023 23:39:46 -0400 Subject: [PATCH 14/24] Fixing discrepancy with unit test --- pandas/tests/io/excel/test_writers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 245419f542985..71ccb66bcfdcc 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1124,7 +1124,6 @@ def test_engine_kwargs(self, engine, path): "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", "xlsxwriter": r"Workbook.__init__() got an unexpected keyword " r"argument 'foo'", - None: r"__init__() got an unexpected keyword argument 'foo'", } # Handle change in error message for openpyxl (write and append mode) @@ -1133,6 +1132,9 @@ def test_engine_kwargs(self, engine, path): "openpyxl" ] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" + elif not engine: + msgs["xlsxwriter"] = "__init__() got an unexpected keyword argument 'foo'" + with pytest.raises(TypeError, match=re.escape(msgs[engine])): df.to_excel( path, From b095e101bd12af5d6f0e08211768ceae83590746 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Mon, 15 May 2023 23:40:15 -0400 Subject: [PATCH 15/24] Fixing discrepancy with unit test --- pandas/tests/io/excel/test_writers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 71ccb66bcfdcc..5b2ee0190a388 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1133,7 +1133,7 @@ def test_engine_kwargs(self, engine, path): ] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" elif not engine: - msgs["xlsxwriter"] = "__init__() got an unexpected keyword argument 'foo'" + msgs["xlsxwriter"] = r"__init__() got an unexpected keyword argument 'foo'" with pytest.raises(TypeError, match=re.escape(msgs[engine])): df.to_excel( From f3dd277bfd85d47e8cfe4453c5d3e0ae6a544fa3 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Tue, 16 May 2023 20:07:34 -0400 Subject: [PATCH 16/24] Updating documentation and syntactical discrepancies based on feedback --- pandas/core/generic.py | 4 ++-- pandas/io/excel/_xlsxwriter.py | 3 ++- pandas/tests/io/excel/test_writers.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index e21e89084fbad..4128b6e3a24c2 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2155,7 +2155,7 @@ def to_excel( inf_rep: str = "inf", freeze_panes: tuple[int, int] | None = None, storage_options: StorageOptions = None, - engine_kwargs: dict | None = None, + engine_kwargs: dict[str, Any] | None = None, ) -> None: """ Write {klass} to an Excel sheet. @@ -2212,7 +2212,7 @@ def to_excel( {storage_options} .. versionadded:: {storage_options_versionadded} - engine_kwargs : dict + engine_kwargs : dict, optional Arbitrary keyword arguments passed to excel engine. See Also diff --git a/pandas/io/excel/_xlsxwriter.py b/pandas/io/excel/_xlsxwriter.py index d7262c2f62d94..fdb23f396cfb2 100644 --- a/pandas/io/excel/_xlsxwriter.py +++ b/pandas/io/excel/_xlsxwriter.py @@ -215,8 +215,9 @@ def __init__( try: self._book = Workbook(self._handles.handle, **engine_kwargs) except TypeError: - self._handles.handle.close() raise + finally: + self._handles.handle.close() @property def book(self): diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 5b2ee0190a388..50018e76cc6d5 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1116,6 +1116,7 @@ def test_bytes_io(self, engine): tm.assert_frame_equal(df, reread_df) def test_engine_kwargs(self, engine, path): + # GH#52368 df = DataFrame([{"A": 1, "B": 2}, {"A": 3, "B": 4}]) msgs = { From bbe1d3be17c132d5484fdb0084387e1c4f92b31d Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Tue, 16 May 2023 21:00:14 -0400 Subject: [PATCH 17/24] Removing finally statement in _xlsxwriter.py to troubleshoot failing unit test --- pandas/io/excel/_xlsxwriter.py | 3 +-- pandas/tests/io/excel/test_writers.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/io/excel/_xlsxwriter.py b/pandas/io/excel/_xlsxwriter.py index fdb23f396cfb2..d7262c2f62d94 100644 --- a/pandas/io/excel/_xlsxwriter.py +++ b/pandas/io/excel/_xlsxwriter.py @@ -215,9 +215,8 @@ def __init__( try: self._book = Workbook(self._handles.handle, **engine_kwargs) except TypeError: - raise - finally: self._handles.handle.close() + raise @property def book(self): diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 50018e76cc6d5..d692f82fa2f2c 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1133,9 +1133,6 @@ def test_engine_kwargs(self, engine, path): "openpyxl" ] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" - elif not engine: - msgs["xlsxwriter"] = r"__init__() got an unexpected keyword argument 'foo'" - with pytest.raises(TypeError, match=re.escape(msgs[engine])): df.to_excel( path, From 010e2243bce3022ae2387d5afbf01e3cb01c86b3 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Tue, 16 May 2023 21:32:15 -0400 Subject: [PATCH 18/24] Fixing discrepancies with unit tests. All local unit tests are passing. --- pandas/tests/io/excel/test_writers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index d692f82fa2f2c..f68919cffb206 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1129,9 +1129,7 @@ def test_engine_kwargs(self, engine, path): # Handle change in error message for openpyxl (write and append mode) if engine == "openpyxl" and os.path.exists(path): - msgs[ - "openpyxl" - ] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" + msgs["openpyxl"] = r"__init__() got an unexpected keyword argument 'foo'" with pytest.raises(TypeError, match=re.escape(msgs[engine])): df.to_excel( From 7e023ad6599f3d38e6361b98f9076e4b35745dc1 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Tue, 16 May 2023 22:06:56 -0400 Subject: [PATCH 19/24] Fixing discrepancies with unit tests. All local unit tests are passing. --- pandas/tests/io/excel/test_writers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index f68919cffb206..547e9bfe762b0 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1123,8 +1123,7 @@ def test_engine_kwargs(self, engine, path): "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword " r"argument 'foo'", "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", - "xlsxwriter": r"Workbook.__init__() got an unexpected keyword " - r"argument 'foo'", + "xlsxwriter": r"__init__() got an unexpected keyword argument 'foo'", } # Handle change in error message for openpyxl (write and append mode) From 5b76c062daeefb39750b403f274e5a33eb5c9065 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Wed, 17 May 2023 19:07:45 -0400 Subject: [PATCH 20/24] Updating error messages in unit tests to ensure that tests are failing on the expected class --- pandas/tests/io/excel/test_writers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 547e9bfe762b0..83e1f657f2767 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1123,12 +1123,15 @@ def test_engine_kwargs(self, engine, path): "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword " r"argument 'foo'", "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", - "xlsxwriter": r"__init__() got an unexpected keyword argument 'foo'", + "xlsxwriter": r"Workbook.__init__() got an unexpected keyword argument " + r"'foo'", } # Handle change in error message for openpyxl (write and append mode) if engine == "openpyxl" and os.path.exists(path): - msgs["openpyxl"] = r"__init__() got an unexpected keyword argument 'foo'" + msgs[ + "openpyxl" + ] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" with pytest.raises(TypeError, match=re.escape(msgs[engine])): df.to_excel( From c014e4ff2465710784c3a4c9bd2e7af3cd077f08 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Wed, 17 May 2023 20:42:04 -0400 Subject: [PATCH 21/24] Updating error messages in unit tests to ensure that tests are failing on the expected class --- pandas/tests/io/excel/test_writers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 83e1f657f2767..7e6f0f76bb9cf 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1136,6 +1136,7 @@ def test_engine_kwargs(self, engine, path): with pytest.raises(TypeError, match=re.escape(msgs[engine])): df.to_excel( path, + engine=engine, engine_kwargs={"foo": "bar"}, ) From e31ec6ac8a16778741d757f088797ba33ce9bde3 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Fri, 19 May 2023 08:03:48 -0400 Subject: [PATCH 22/24] Updating error messages in unit tests to ensure that tests are failing on the expected class --- pandas/tests/io/excel/test_writers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 7e6f0f76bb9cf..4a22c2a32dd72 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -11,6 +11,7 @@ import numpy as np import pytest +from pandas.compat._constants import PY310 import pandas.util._test_decorators as td import pandas as pd @@ -1127,11 +1128,13 @@ def test_engine_kwargs(self, engine, path): r"'foo'", } + if PY310: + msgs["xlsxwriter"] = "__init__() got an unexpected keyword argument " + r"'foo'" + # Handle change in error message for openpyxl (write and append mode) - if engine == "openpyxl" and os.path.exists(path): - msgs[ - "openpyxl" - ] = r"Workbook.__init__() got an unexpected keyword argument 'foo'" + if engine == "openpyxl" and os.path.exists(path) and PY310: + msgs["openpyxl"] = r"__init__() got an unexpected keyword argument 'foo'" with pytest.raises(TypeError, match=re.escape(msgs[engine])): df.to_excel( From 50ef9f8eb6c3a840d48d48c8ede7a9e4124b2022 Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Fri, 19 May 2023 08:45:45 -0400 Subject: [PATCH 23/24] Updating error messages in unit tests to ensure that tests are failing on the expected class --- pandas/tests/io/excel/test_writers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 4a22c2a32dd72..ca2308b2ab014 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1129,8 +1129,7 @@ def test_engine_kwargs(self, engine, path): } if PY310: - msgs["xlsxwriter"] = "__init__() got an unexpected keyword argument " - r"'foo'" + msgs["xlsxwriter"] = "__init__() got an unexpected keyword argument 'foo'" # Handle change in error message for openpyxl (write and append mode) if engine == "openpyxl" and os.path.exists(path) and PY310: From cbad00f6334399a63bb5feb323eb4650486e564a Mon Sep 17 00:00:00 2001 From: Richard Howe Date: Fri, 19 May 2023 09:57:26 -0400 Subject: [PATCH 24/24] Updating error messages in unit tests to ensure that tests are failing on the expected class --- pandas/tests/io/excel/test_writers.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index ca2308b2ab014..0560e12a00bf5 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1123,17 +1123,23 @@ def test_engine_kwargs(self, engine, path): msgs = { "odf": r"OpenDocumentSpreadsheet() got an unexpected keyword " r"argument 'foo'", - "openpyxl": r"load_workbook() got an unexpected keyword argument 'foo'", - "xlsxwriter": r"Workbook.__init__() got an unexpected keyword argument " - r"'foo'", + "openpyxl": r"__init__() got an unexpected keyword argument 'foo'", + "xlsxwriter": r"__init__() got an unexpected keyword argument 'foo'", } if PY310: - msgs["xlsxwriter"] = "__init__() got an unexpected keyword argument 'foo'" + msgs[ + "openpyxl" + ] = "Workbook.__init__() got an unexpected keyword argument 'foo'" + msgs[ + "xlsxwriter" + ] = "Workbook.__init__() got an unexpected keyword argument 'foo'" # Handle change in error message for openpyxl (write and append mode) - if engine == "openpyxl" and os.path.exists(path) and PY310: - msgs["openpyxl"] = r"__init__() got an unexpected keyword argument 'foo'" + if engine == "openpyxl" and not os.path.exists(path): + msgs[ + "openpyxl" + ] = r"load_workbook() got an unexpected keyword argument 'foo'" with pytest.raises(TypeError, match=re.escape(msgs[engine])): df.to_excel(