From b18af6f37a660b69e64bca2fae573a7e539d58aa Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Tue, 7 Sep 2021 17:22:54 +0200 Subject: [PATCH 01/15] passes through now for openpyxl and xlwt --- pandas/io/excel/_openpyxl.py | 2 +- pandas/io/excel/_xlwt.py | 2 +- pandas/tests/io/excel/test_openpyxl.py | 7 ++++--- pandas/tests/io/excel/test_xlwt.py | 11 ++++++----- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index d499f1a5ea89f..2adbcd71ff65b 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -68,7 +68,7 @@ def __init__( else: # Create workbook object with default optimized_write=True. - self.book = Workbook() + self.book = Workbook(**engine_kwargs) if self.book.worksheets: self.book.remove(self.book.worksheets[0]) diff --git a/pandas/io/excel/_xlwt.py b/pandas/io/excel/_xlwt.py index 4dadf64b44515..a74c03f330cd9 100644 --- a/pandas/io/excel/_xlwt.py +++ b/pandas/io/excel/_xlwt.py @@ -53,7 +53,7 @@ def __init__( if encoding is None: encoding = "ascii" - self.book = xlwt.Workbook(encoding=encoding) + self.book = xlwt.Workbook(encoding=encoding, **engine_kwargs) self.fm_datetime = xlwt.easyxf(num_format_str=self.datetime_format) self.fm_date = xlwt.easyxf(num_format_str=self.date_format) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index cd773957c9043..2253f4681c288 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -98,14 +98,15 @@ def test_kwargs(ext, write_only): DataFrame().to_excel(writer) -@pytest.mark.parametrize("write_only", [True, False]) -def test_engine_kwargs(ext, write_only): +@pytest.mark.parametrize("iso_dates", [True, False]) +def test_engine_kwargs(ext, iso_dates): # GH 42286 # openpyxl doesn't utilize kwargs, only test that supplying a engine_kwarg works - engine_kwargs = {"write_only": write_only} + engine_kwargs = {"iso_dates": iso_dates} with tm.ensure_clean(ext) as f: with ExcelWriter(f, engine="openpyxl", engine_kwargs=engine_kwargs) as writer: # ExcelWriter won't allow us to close without writing something + assert writer.book.iso_dates == iso_dates DataFrame().to_excel(writer) diff --git a/pandas/tests/io/excel/test_xlwt.py b/pandas/tests/io/excel/test_xlwt.py index c58b9763f9618..cbe99d2d35bd4 100644 --- a/pandas/tests/io/excel/test_xlwt.py +++ b/pandas/tests/io/excel/test_xlwt.py @@ -114,12 +114,13 @@ def test_kwargs(ext, write_only): DataFrame().to_excel(writer) -@pytest.mark.parametrize("write_only", [True, False]) -def test_engine_kwargs(ext, write_only): +@pytest.mark.parametrize("style_compression", [0, 2]) +def test_engine_kwargs(ext, style_compression): # GH 42286 - # xlwt doesn't utilize kwargs, only test that supplying a engine_kwarg works - engine_kwargs = {"write_only": write_only} + # + engine_kwargs = {"style_compression": style_compression} with tm.ensure_clean(ext) as f: - with ExcelWriter(f, engine="openpyxl", engine_kwargs=engine_kwargs) as writer: + with ExcelWriter(f, engine="xlwt", engine_kwargs=engine_kwargs) as writer: # xlwt won't allow us to close without writing something + assert writer.book._Workbook__styles.style_compression == style_compression DataFrame().to_excel(writer) From 7c439c8ce67e85bd007bc3d917a5646b91c53f5c Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Tue, 7 Sep 2021 17:28:55 +0200 Subject: [PATCH 02/15] added docs --- pandas/io/excel/_base.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index 52d1e1c83d3e6..ef5d37b370320 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -691,7 +691,8 @@ class ExcelWriter(metaclass=abc.ABCMeta): .. versionadded:: 1.3.0 engine_kwargs : dict, optional - Keyword arguments to be passed into the engine. + Keyword arguments to be passed into the engine. Only works for + xlsxwriter, openpyxl and xlwt. Does not work in append mode. .. versionadded:: 1.3.0 **kwargs : dict, optional @@ -771,6 +772,15 @@ class ExcelWriter(metaclass=abc.ABCMeta): ... with zf.open("filename.xlsx", "w") as buffer: ... with pd.ExcelWriter(buffer) as writer: ... df.to_excel(writer) + + You can specify additional arguments to the underlying engine: + + >>> with ExcelWriter( + ... "path_to_file.xlsx", + ... engine="xlsxwriter", + ... engine_kwargs={"options":{"nan_inf_to_errors":True}} + ... ) as writer: + ... df.to_excel(writer) """ # Defining an ExcelWriter implementation (see abstract methods for more...) From d214487781d169ebffa2444134b5a2f7cd395c3a Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Wed, 8 Sep 2021 10:22:52 +0200 Subject: [PATCH 03/15] BUG: GH43445 added append mode, tests --- pandas/io/excel/_base.py | 15 +++++++++++-- pandas/io/excel/_openpyxl.py | 2 +- pandas/tests/io/excel/test_openpyxl.py | 29 +++++++++++++++++++------- pandas/tests/io/excel/test_xlwt.py | 15 ++++++------- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index ef5d37b370320..0a47706c39d54 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -691,8 +691,7 @@ class ExcelWriter(metaclass=abc.ABCMeta): .. versionadded:: 1.3.0 engine_kwargs : dict, optional - Keyword arguments to be passed into the engine. Only works for - xlsxwriter, openpyxl and xlwt. Does not work in append mode. + Keyword arguments to be passed into the engine. Not supported for odswriter. .. versionadded:: 1.3.0 **kwargs : dict, optional @@ -781,6 +780,18 @@ class ExcelWriter(metaclass=abc.ABCMeta): ... engine_kwargs={"options":{"nan_inf_to_errors":True}} ... ) as writer: ... df.to_excel(writer) + + In append mode, ``engine_kwargs`` are passed through to + openpyxl's ``load_workbook``: + + >>> with ExcelWriter( + ... "path_to_file.xlsx", + ... engine="openpyxl", + ... mode="a", + ... engine_kwargs={"keep_vba":True} + ... ) as writer: + ... df.to_excel(writer) + ) """ # Defining an ExcelWriter implementation (see abstract methods for more...) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 2adbcd71ff65b..62c7991148498 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -62,7 +62,7 @@ def __init__( if "r+" in self.mode: # Load from existing workbook from openpyxl import load_workbook - self.book = load_workbook(self.handles.handle) + self.book = load_workbook(self.handles.handle, **engine_kwargs) self.handles.handle.seek(0) self.sheets = {name: self.book[name] for name in self.book.sheetnames} diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 2253f4681c288..8f9b9ffe6154c 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -85,31 +85,46 @@ def test_write_cells_merge_styled(ext): assert xcell_a2.font == openpyxl_sty_merged -@pytest.mark.parametrize("write_only", [True, False]) -def test_kwargs(ext, write_only): +@pytest.mark.parametrize("iso_dates", [True, False]) +def test_kwargs(ext, iso_dates): # GH 42286 # openpyxl doesn't utilize kwargs, only test that supplying a kwarg works - kwargs = {"write_only": write_only} + kwargs = {"iso_dates": iso_dates} with tm.ensure_clean(ext) as f: msg = re.escape("Use of **kwargs is deprecated") with tm.assert_produces_warning(FutureWarning, match=msg): with ExcelWriter(f, engine="openpyxl", **kwargs) as writer: + assert writer.book.iso_dates == iso_dates # ExcelWriter won't allow us to close without writing something DataFrame().to_excel(writer) @pytest.mark.parametrize("iso_dates", [True, False]) -def test_engine_kwargs(ext, iso_dates): - # GH 42286 - # openpyxl doesn't utilize kwargs, only test that supplying a engine_kwarg works +def test_engine_kwargs_write(ext, iso_dates): + # GH 42286 GH 43445 engine_kwargs = {"iso_dates": iso_dates} with tm.ensure_clean(ext) as f: with ExcelWriter(f, engine="openpyxl", engine_kwargs=engine_kwargs) as writer: - # ExcelWriter won't allow us to close without writing something assert writer.book.iso_dates == iso_dates + # ExcelWriter won't allow us to close without writing something DataFrame().to_excel(writer) +def test_engine_kwargs_append(ext): + # GH 43445 + # only read_only=True will give something easy that can be verified (maybe + # keep_links or data_only could also work, but that'd be more complicated) + # arguments are passed through to load_workbook() + engine_kwargs = {"read_only": True} + with tm.ensure_clean(ext) as f: + DataFrame(["hello", "world"]).to_excel(f) + with pytest.raises(TypeError, match=re.escape("Workbook is read-only")): + with ExcelWriter( + f, engine="openpyxl", mode="a", engine_kwargs=engine_kwargs + ) as writer: + DataFrame(["goodbye", "world"]).to_excel(writer) + + @pytest.mark.parametrize( "mode,expected", [("w", ["baz"]), ("a", ["foo", "bar", "baz"])] ) diff --git a/pandas/tests/io/excel/test_xlwt.py b/pandas/tests/io/excel/test_xlwt.py index cbe99d2d35bd4..ec333defd85ac 100644 --- a/pandas/tests/io/excel/test_xlwt.py +++ b/pandas/tests/io/excel/test_xlwt.py @@ -101,15 +101,17 @@ def test_option_xls_writer_deprecated(ext): options.io.excel.xls.writer = "xlwt" -@pytest.mark.parametrize("write_only", [True, False]) -def test_kwargs(ext, write_only): +@pytest.mark.parametrize("style_compression", [0, 2]) +def test_kwargs(ext, style_compression): # GH 42286 - # xlwt doesn't utilize kwargs, only test that supplying a kwarg works - kwargs = {"write_only": write_only} + kwargs = {"style_compression": style_compression} with tm.ensure_clean(ext) as f: msg = re.escape("Use of **kwargs is deprecated") with tm.assert_produces_warning(FutureWarning, match=msg): - with ExcelWriter(f, engine="openpyxl", **kwargs) as writer: + with ExcelWriter(f, engine="xlwt", **kwargs) as writer: + assert ( + writer.book._Workbook__styles.style_compression == style_compression + ) # xlwt won't allow us to close without writing something DataFrame().to_excel(writer) @@ -117,10 +119,9 @@ def test_kwargs(ext, write_only): @pytest.mark.parametrize("style_compression", [0, 2]) def test_engine_kwargs(ext, style_compression): # GH 42286 - # engine_kwargs = {"style_compression": style_compression} with tm.ensure_clean(ext) as f: with ExcelWriter(f, engine="xlwt", engine_kwargs=engine_kwargs) as writer: - # xlwt won't allow us to close without writing something assert writer.book._Workbook__styles.style_compression == style_compression + # xlwt won't allow us to close without writing something DataFrame().to_excel(writer) From e7c2b6e0c10fa047658d46214de666bca3e28446 Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Wed, 8 Sep 2021 12:10:39 +0200 Subject: [PATCH 04/15] BUG: GH43445 typo --- pandas/io/excel/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index 0a47706c39d54..8ffa53cb8859f 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -791,7 +791,6 @@ class ExcelWriter(metaclass=abc.ABCMeta): ... engine_kwargs={"keep_vba":True} ... ) as writer: ... df.to_excel(writer) - ) """ # Defining an ExcelWriter implementation (see abstract methods for more...) From a4a29ee07d29636e0fc73f7bbb197dd0df0a2b5d Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Thu, 16 Sep 2021 23:01:06 +0200 Subject: [PATCH 05/15] BUG: GH43445 docs and opendocumentwriter --- pandas/io/excel/_base.py | 9 ++++++++- pandas/io/excel/_odswriter.py | 9 +++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index 8ffa53cb8859f..ca2956ea225be 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -691,7 +691,14 @@ class ExcelWriter(metaclass=abc.ABCMeta): .. versionadded:: 1.3.0 engine_kwargs : dict, optional - Keyword arguments to be passed into the engine. Not supported for odswriter. + Keyword arguments to be passed into the engine. These will be passed to + the following functions of the respective engines: + + * odswriter: ``odf.opendocument.OpenDocumentSpreadsheet(**engine_kwargs)`` + * xlsxwriter: ``xlsxwriter.Workbook(file, **engine_kwargs)`` + * xlwt: ``xlwt.Workbook(encoding, **engine_kwargs)`` + * openpyxl (write mode): ``openpyxl.Workbook(**engine_kwargs)`` + * openpyxl (append mode): ``openpyxl.load_workbook(file, **engine_kwargs)`` .. versionadded:: 1.3.0 **kwargs : dict, optional diff --git a/pandas/io/excel/_odswriter.py b/pandas/io/excel/_odswriter.py index fa2779b01d681..add95c58cd809 100644 --- a/pandas/io/excel/_odswriter.py +++ b/pandas/io/excel/_odswriter.py @@ -11,7 +11,10 @@ from pandas._typing import StorageOptions from pandas.io.excel._base import ExcelWriter -from pandas.io.excel._util import validate_freeze_panes +from pandas.io.excel._util import ( + combine_kwargs, + validate_freeze_panes, +) from pandas.io.formats.excel import ExcelCell @@ -44,7 +47,9 @@ def __init__( engine_kwargs=engine_kwargs, ) - self.book = OpenDocumentSpreadsheet() + engine_kwargs = combine_kwargs(engine_kwargs, kwargs) + + self.book = OpenDocumentSpreadsheet(**engine_kwargs) self._style_dict: dict[str, str] = {} def save(self) -> None: From 44653f9318c4ca5d92aa9a6a2b018118eeabcace Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Thu, 16 Sep 2021 23:28:15 +0200 Subject: [PATCH 06/15] BUG: GH43445 tests for odswriter --- pandas/tests/io/excel/test_odswriter.py | 47 +++++++++++++++++-------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/pandas/tests/io/excel/test_odswriter.py b/pandas/tests/io/excel/test_odswriter.py index 4bf6051fd36ef..0e6d1dac55506 100644 --- a/pandas/tests/io/excel/test_odswriter.py +++ b/pandas/tests/io/excel/test_odswriter.py @@ -19,23 +19,40 @@ def test_write_append_mode_raises(ext): ExcelWriter(f, engine="odf", mode="a") -@pytest.mark.parametrize("nan_inf_to_errors", [True, False]) -def test_kwargs(ext, nan_inf_to_errors): +def test_kwargs(ext): # GH 42286 - # odswriter doesn't utilize kwargs, nothing to check except that it works - kwargs = {"options": {"nan_inf_to_errors": nan_inf_to_errors}} + # GH 43445 + # test for error: OpenDocumentSpreadsheet does not accept any arguments + kwargs = {"kwarg": 1} with tm.ensure_clean(ext) as f: msg = re.escape("Use of **kwargs is deprecated") - with tm.assert_produces_warning(FutureWarning, match=msg): - with ExcelWriter(f, engine="odf", **kwargs) as _: - pass - - -@pytest.mark.parametrize("nan_inf_to_errors", [True, False]) -def test_engine_kwargs(ext, nan_inf_to_errors): + error = re.escape( + "OpenDocumentSpreadsheet() got an unexpected keyword argument 'kwarg'" + ) + with pytest.raises( + TypeError, + match=error, + ): + with tm.assert_produces_warning(FutureWarning, match=msg): + with ExcelWriter(f, engine="odf", **kwargs) as _: + pass + + +@pytest.mark.parametrize("engine_kwargs", [None, {"kwarg": 1}]) +def test_engine_kwargs(ext, engine_kwargs): # GH 42286 - # odswriter doesn't utilize engine_kwargs, nothing to check except that it works - engine_kwargs = {"options": {"nan_inf_to_errors": nan_inf_to_errors}} + # GH 43445 + # test for error: OpenDocumentSpreadsheet does not accept any arguments with tm.ensure_clean(ext) as f: - with ExcelWriter(f, engine="odf", engine_kwargs=engine_kwargs) as _: - pass + if engine_kwargs is not None: + error = re.escape( + "OpenDocumentSpreadsheet() got an unexpected keyword argument 'kwarg'" + ) + with pytest.raises( + TypeError, + match=error, + ): + ExcelWriter(f, engine="odf", engine_kwargs=engine_kwargs) + else: + with ExcelWriter(f, engine="odf", engine_kwargs=engine_kwargs) as _: + pass From c94561b40af3ccd8a18d9ecf701546a76c0ff241 Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Mon, 20 Sep 2021 17:09:52 +0200 Subject: [PATCH 07/15] BUG: GH 43445 doctests --- pandas/io/excel/_base.py | 18 +++++++++--------- pandas/tests/io/excel/test_openpyxl.py | 3 +-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index ca2956ea225be..505e5c51fd0b0 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -728,14 +728,14 @@ class ExcelWriter(metaclass=abc.ABCMeta): Default usage: >>> df = pd.DataFrame([["ABC", "XYZ"]], columns=["Foo", "Bar"]) - >>> with ExcelWriter("path_to_file.xlsx") as writer: + >>> with pd.ExcelWriter("path_to_file.xlsx") as writer: ... df.to_excel(writer) To write to separate sheets in a single file: >>> df1 = pd.DataFrame([["AAA", "BBB"]], columns=["Spam", "Egg"]) >>> df2 = pd.DataFrame([["ABC", "XYZ"]], columns=["Foo", "Bar"]) - >>> with ExcelWriter("path_to_file.xlsx") as writer: + >>> with pd.ExcelWriter("path_to_file.xlsx") as writer: ... df1.to_excel(writer, sheet_name="Sheet1") ... df2.to_excel(writer, sheet_name="Sheet2") @@ -750,7 +750,7 @@ class ExcelWriter(metaclass=abc.ABCMeta): ... index=["Date", "Datetime"], ... columns=["X", "Y"], ... ) - >>> with ExcelWriter( + >>> with pd.ExcelWriter( ... "path_to_file.xlsx", ... date_format="YYYY-MM-DD", ... datetime_format="YYYY-MM-DD HH:MM:SS" @@ -759,7 +759,7 @@ class ExcelWriter(metaclass=abc.ABCMeta): You can also append to an existing Excel file: - >>> with ExcelWriter("path_to_file.xlsx", mode="a", engine="openpyxl") as writer: + >>> with pd.ExcelWriter("path_to_file.xlsx", mode="a", engine="openpyxl") as writer: ... df.to_excel(writer, sheet_name="Sheet3") You can store Excel file in RAM: @@ -781,23 +781,23 @@ class ExcelWriter(metaclass=abc.ABCMeta): You can specify additional arguments to the underlying engine: - >>> with ExcelWriter( + >>> with pd.ExcelWriter( ... "path_to_file.xlsx", ... engine="xlsxwriter", - ... engine_kwargs={"options":{"nan_inf_to_errors":True}} + ... engine_kwargs={"options": {"nan_inf_to_errors": True}} ... ) as writer: ... df.to_excel(writer) In append mode, ``engine_kwargs`` are passed through to openpyxl's ``load_workbook``: - >>> with ExcelWriter( + >>> with pd.ExcelWriter( ... "path_to_file.xlsx", ... engine="openpyxl", ... mode="a", - ... engine_kwargs={"keep_vba":True} + ... engine_kwargs={"keep_vba": True} ... ) as writer: - ... df.to_excel(writer) + ... df.to_excel(writer, sheet_name="Sheet2") """ # Defining an ExcelWriter implementation (see abstract methods for more...) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 8f9b9ffe6154c..8eac0ba7c730b 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -87,8 +87,7 @@ def test_write_cells_merge_styled(ext): @pytest.mark.parametrize("iso_dates", [True, False]) def test_kwargs(ext, iso_dates): - # GH 42286 - # openpyxl doesn't utilize kwargs, only test that supplying a kwarg works + # GH 42286 GH 43445 kwargs = {"iso_dates": iso_dates} with tm.ensure_clean(ext) as f: msg = re.escape("Use of **kwargs is deprecated") From 4708e3e02342a4a18ffee0eeb315ddf64fd7c7cd Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Thu, 4 Nov 2021 15:43:05 +0100 Subject: [PATCH 08/15] GH 43445: improved test_engine_kwargs_append --- pandas/tests/io/excel/test_openpyxl.py | 31 +++++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 8eac0ba7c730b..2e54fa84bb1a8 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -109,19 +109,38 @@ def test_engine_kwargs_write(ext, iso_dates): DataFrame().to_excel(writer) -def test_engine_kwargs_append(ext): +@pytest.mark.parametrize( + "engine_kwargs", [{"data_only": True}, {"keep_vba": True}, {"keep_links": False}] +) +def test_engine_kwargs_append(ext, engine_kwargs): # GH 43445 # only read_only=True will give something easy that can be verified (maybe # keep_links or data_only could also work, but that'd be more complicated) # arguments are passed through to load_workbook() - engine_kwargs = {"read_only": True} with tm.ensure_clean(ext) as f: DataFrame(["hello", "world"]).to_excel(f) - with pytest.raises(TypeError, match=re.escape("Workbook is read-only")): + with ExcelWriter(f, engine="openpyxl", mode="a", engine_kwargs=engine_kwargs): + pass + # DataFrame(["goodbye", "world"]).to_excel(writer) + + +def test_engine_kwargs_append_invalid(ext): + # GH 43445 + # test whether an invalid engine kwargs actually raises + with tm.ensure_clean(ext) as f: + DataFrame(["hello", "world"]).to_excel(f) + with pytest.raises( + TypeError, + match=re.escape( + "load_workbook() got an unexpected keyword argument 'apple_banana'" + ), + ): with ExcelWriter( - f, engine="openpyxl", mode="a", engine_kwargs=engine_kwargs - ) as writer: - DataFrame(["goodbye", "world"]).to_excel(writer) + f, engine="openpyxl", mode="a", engine_kwargs={"apple_banana": "fruit"} + ): + # not sure if we have to do something here? + # DataFrame(["good"]) + pass @pytest.mark.parametrize( From fb949f5590061ad46f4eeddf5d22a5d79df5c0e0 Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Thu, 4 Nov 2021 15:50:24 +0100 Subject: [PATCH 09/15] GH 43445: added test that actually tests --- pandas/tests/io/excel/test_openpyxl.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 2e54fa84bb1a8..fc70bd24391f0 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -124,6 +124,21 @@ def test_engine_kwargs_append(ext, engine_kwargs): # DataFrame(["goodbye", "world"]).to_excel(writer) +@pytest.mark.parametrize("data_only,B2", [(True, 0), (False, "=1+1")]) +def test_engine_kwargs_append_keep_links(ext, data_only, B2): + # GH 43445 + # tests whether the keep_links engine_kwarg actually works well for + # openpyxl's load_workbook + # not sure if we have to test for this though, since it also relies a bit on + # functionality of openpyxl + with tm.ensure_clean(ext) as f: + DataFrame(["=1+1"]).to_excel(f) + with ExcelWriter( + f, engine="openpyxl", mode="a", engine_kwargs={"data_only": data_only} + ) as writer: + assert writer.sheets["Sheet1"]["B2"].value == B2 + + def test_engine_kwargs_append_invalid(ext): # GH 43445 # test whether an invalid engine kwargs actually raises From 916f0770035169332002f8380361a6344b65e1de Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Thu, 4 Nov 2021 16:04:48 +0100 Subject: [PATCH 10/15] GH 43445: add writing to ExcelWriter --- pandas/tests/io/excel/test_openpyxl.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index fc70bd24391f0..904ed5dff5e56 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -119,9 +119,10 @@ def test_engine_kwargs_append(ext, engine_kwargs): # arguments are passed through to load_workbook() with tm.ensure_clean(ext) as f: DataFrame(["hello", "world"]).to_excel(f) - with ExcelWriter(f, engine="openpyxl", mode="a", engine_kwargs=engine_kwargs): - pass - # DataFrame(["goodbye", "world"]).to_excel(writer) + with ExcelWriter( + f, engine="openpyxl", mode="a", engine_kwargs=engine_kwargs + ) as writer: + DataFrame(["goodbye", "world"]).to_excel(writer, sheet_name="Sheet2") @pytest.mark.parametrize("data_only,B2", [(True, 0), (False, "=1+1")]) @@ -137,6 +138,8 @@ def test_engine_kwargs_append_keep_links(ext, data_only, B2): f, engine="openpyxl", mode="a", engine_kwargs={"data_only": data_only} ) as writer: assert writer.sheets["Sheet1"]["B2"].value == B2 + # ExcelWriter needs us to writer something to close properly? + DataFrame().to_excel(writer, sheet_name="Sheet2") def test_engine_kwargs_append_invalid(ext): @@ -152,10 +155,9 @@ def test_engine_kwargs_append_invalid(ext): ): with ExcelWriter( f, engine="openpyxl", mode="a", engine_kwargs={"apple_banana": "fruit"} - ): + ) as writer: # not sure if we have to do something here? - # DataFrame(["good"]) - pass + DataFrame(["good"]).to_excel(writer, sheet_name="Sheet2") @pytest.mark.parametrize( From 97589eaf25e6a64e3564c22e362424f5fd6192b3 Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Sun, 7 Nov 2021 13:50:13 +0100 Subject: [PATCH 11/15] BUG: GH43445 improved order, whatsnew --- doc/source/whatsnew/v1.4.0.rst | 2 +- pandas/io/excel/_base.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index 4d0dee01f05c1..f3d64fa46f2ec 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -571,7 +571,7 @@ I/O - Bug in :func:`json_normalize` where multi-character ``sep`` parameter is incorrectly prefixed to every key (:issue:`43831`) - Bug in :func:`read_csv` with :code:`float_precision="round_trip"` which did not skip initial/trailing whitespace (:issue:`43713`) - Bug in dumping/loading a :class:`DataFrame` with ``yaml.dump(frame)`` (:issue:`42748`) -- +- Bug in :class:`ExcelWriter`, where ``engine_kwargs`` were not passed through to all engines (:issue:`43442`) Period ^^^^^^ diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index 22433fa5e83c0..1409050153ceb 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -703,11 +703,10 @@ class ExcelWriter(metaclass=abc.ABCMeta): Keyword arguments to be passed into the engine. These will be passed to the following functions of the respective engines: - * odswriter: ``odf.opendocument.OpenDocumentSpreadsheet(**engine_kwargs)`` * xlsxwriter: ``xlsxwriter.Workbook(file, **engine_kwargs)`` - * xlwt: ``xlwt.Workbook(encoding, **engine_kwargs)`` * openpyxl (write mode): ``openpyxl.Workbook(**engine_kwargs)`` * openpyxl (append mode): ``openpyxl.load_workbook(file, **engine_kwargs)`` + * odswriter: ``odf.opendocument.OpenDocumentSpreadsheet(**engine_kwargs)`` .. versionadded:: 1.3.0 **kwargs : dict, optional From 8a7e79d21b59c189891eeced944765c12a3e72d0 Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Tue, 9 Nov 2021 09:53:30 +0100 Subject: [PATCH 12/15] ENH: GH43445 last fixes --- pandas/tests/io/excel/test_openpyxl.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 904ed5dff5e56..88da360e2a75d 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -114,9 +114,6 @@ def test_engine_kwargs_write(ext, iso_dates): ) def test_engine_kwargs_append(ext, engine_kwargs): # GH 43445 - # only read_only=True will give something easy that can be verified (maybe - # keep_links or data_only could also work, but that'd be more complicated) - # arguments are passed through to load_workbook() with tm.ensure_clean(ext) as f: DataFrame(["hello", "world"]).to_excel(f) with ExcelWriter( @@ -125,19 +122,17 @@ def test_engine_kwargs_append(ext, engine_kwargs): DataFrame(["goodbye", "world"]).to_excel(writer, sheet_name="Sheet2") -@pytest.mark.parametrize("data_only,B2", [(True, 0), (False, "=1+1")]) -def test_engine_kwargs_append_keep_links(ext, data_only, B2): +@pytest.mark.parametrize("data_only, expected", [(True, 0), (False, "=1+1")]) +def test_engine_kwargs_append_keep_links(ext, data_only, expected): # GH 43445 # tests whether the keep_links engine_kwarg actually works well for # openpyxl's load_workbook - # not sure if we have to test for this though, since it also relies a bit on - # functionality of openpyxl with tm.ensure_clean(ext) as f: DataFrame(["=1+1"]).to_excel(f) with ExcelWriter( f, engine="openpyxl", mode="a", engine_kwargs={"data_only": data_only} ) as writer: - assert writer.sheets["Sheet1"]["B2"].value == B2 + assert writer.sheets["Sheet1"]["B2"].value == expected # ExcelWriter needs us to writer something to close properly? DataFrame().to_excel(writer, sheet_name="Sheet2") From 5da52543063da922daaf808bd33f46461a358a1e Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Wed, 10 Nov 2021 15:45:27 +0100 Subject: [PATCH 13/15] ENH: GH43445 briefified test --- pandas/tests/io/excel/test_openpyxl.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 88da360e2a75d..04dd60cb0f6f7 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -109,15 +109,12 @@ def test_engine_kwargs_write(ext, iso_dates): DataFrame().to_excel(writer) -@pytest.mark.parametrize( - "engine_kwargs", [{"data_only": True}, {"keep_vba": True}, {"keep_links": False}] -) -def test_engine_kwargs_append(ext, engine_kwargs): +def test_engine_kwargs_append(ext): # GH 43445 with tm.ensure_clean(ext) as f: DataFrame(["hello", "world"]).to_excel(f) with ExcelWriter( - f, engine="openpyxl", mode="a", engine_kwargs=engine_kwargs + f, engine="openpyxl", mode="a", engine_kwargs={"keep_vba": True} ) as writer: DataFrame(["goodbye", "world"]).to_excel(writer, sheet_name="Sheet2") From f90e7db600ba803093b63aa6ca891a389c6e93fb Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Fri, 12 Nov 2021 08:36:53 +0100 Subject: [PATCH 14/15] BUG: GH43445 typographical error --- pandas/tests/io/excel/test_openpyxl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 04dd60cb0f6f7..0684e911eb63f 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -120,7 +120,7 @@ def test_engine_kwargs_append(ext): @pytest.mark.parametrize("data_only, expected", [(True, 0), (False, "=1+1")]) -def test_engine_kwargs_append_keep_links(ext, data_only, expected): +def test_engine_kwargs_append_data_only(ext, data_only, expected): # GH 43445 # tests whether the keep_links engine_kwarg actually works well for # openpyxl's load_workbook From f8a8fae77c8601a01891bd1db2c9eeed7550e014 Mon Sep 17 00:00:00 2001 From: Joeperdefloep Date: Mon, 15 Nov 2021 16:23:24 +0100 Subject: [PATCH 15/15] BUG: GH43445 finalized tests --- pandas/tests/io/excel/test_openpyxl.py | 38 ++++++++++---------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 0684e911eb63f..1bc96da05f851 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -109,20 +109,28 @@ def test_engine_kwargs_write(ext, iso_dates): DataFrame().to_excel(writer) -def test_engine_kwargs_append(ext): +def test_engine_kwargs_append_invalid(ext): # GH 43445 + # test whether an invalid engine kwargs actually raises with tm.ensure_clean(ext) as f: DataFrame(["hello", "world"]).to_excel(f) - with ExcelWriter( - f, engine="openpyxl", mode="a", engine_kwargs={"keep_vba": True} - ) as writer: - DataFrame(["goodbye", "world"]).to_excel(writer, sheet_name="Sheet2") + with pytest.raises( + TypeError, + match=re.escape( + "load_workbook() got an unexpected keyword argument 'apple_banana'" + ), + ): + with ExcelWriter( + f, engine="openpyxl", mode="a", engine_kwargs={"apple_banana": "fruit"} + ) as writer: + # ExcelWriter needs us to write something to close properly + DataFrame(["good"]).to_excel(writer, sheet_name="Sheet2") @pytest.mark.parametrize("data_only, expected", [(True, 0), (False, "=1+1")]) def test_engine_kwargs_append_data_only(ext, data_only, expected): # GH 43445 - # tests whether the keep_links engine_kwarg actually works well for + # tests whether the data_only engine_kwarg actually works well for # openpyxl's load_workbook with tm.ensure_clean(ext) as f: DataFrame(["=1+1"]).to_excel(f) @@ -134,24 +142,6 @@ def test_engine_kwargs_append_data_only(ext, data_only, expected): DataFrame().to_excel(writer, sheet_name="Sheet2") -def test_engine_kwargs_append_invalid(ext): - # GH 43445 - # test whether an invalid engine kwargs actually raises - with tm.ensure_clean(ext) as f: - DataFrame(["hello", "world"]).to_excel(f) - with pytest.raises( - TypeError, - match=re.escape( - "load_workbook() got an unexpected keyword argument 'apple_banana'" - ), - ): - with ExcelWriter( - f, engine="openpyxl", mode="a", engine_kwargs={"apple_banana": "fruit"} - ) as writer: - # not sure if we have to do something here? - DataFrame(["good"]).to_excel(writer, sheet_name="Sheet2") - - @pytest.mark.parametrize( "mode,expected", [("w", ["baz"]), ("a", ["foo", "bar", "baz"])] )