From a641cce180342b8ca4d0db7b7683dc9439b645bb Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Fri, 14 Jan 2022 19:10:41 -0500 Subject: [PATCH 1/6] ENH: Allow safe access to `.book` in ExcelWriter --- pandas/io/excel/_base.py | 1 - pandas/io/excel/_odswriter.py | 12 +++++++++++- pandas/io/excel/_openpyxl.py | 8 ++++---- pandas/io/excel/_xlsxwriter.py | 6 ++---- pandas/io/excel/_xlwt.py | 5 +++++ pandas/tests/io/excel/test_odswriter.py | 10 ++++++++++ pandas/tests/io/excel/test_openpyxl.py | 9 +++++++++ pandas/tests/io/excel/test_xlwt.py | 9 +++++++++ 8 files changed, 50 insertions(+), 10 deletions(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index 3e1df9325713b..f22ff4b55eadb 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -1112,7 +1112,6 @@ def __init__( self.handles = get_handle( path, mode, storage_options=storage_options, is_text=False ) - self.sheets: dict[str, Any] = {} self.cur_sheet = None if date_format is None: diff --git a/pandas/io/excel/_odswriter.py b/pandas/io/excel/_odswriter.py index 0f31991ee29e9..e30548f220bbc 100644 --- a/pandas/io/excel/_odswriter.py +++ b/pandas/io/excel/_odswriter.py @@ -58,6 +58,16 @@ def __init__( self.book = OpenDocumentSpreadsheet(**engine_kwargs) self._style_dict: dict[str, str] = {} + @property + def sheets(self) -> dict[str, Any]: + from odf.table import Table + + result = { + sheet.getAttribute("name"): sheet + for sheet in self.book.getElementsByType(Table) + } + return result + def save(self) -> None: """ Save workbook to disk. @@ -91,7 +101,7 @@ def write_cells( wks = self.sheets[sheet_name] else: wks = Table(name=sheet_name) - self.sheets[sheet_name] = wks + self.book.spreadsheet.addElement(wks) if validate_freeze_panes(freeze_panes): freeze_panes = cast(Tuple[int, int], freeze_panes) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 88a25d1c1e6ef..c91ee6398c57d 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -68,8 +68,6 @@ def __init__( self.book = load_workbook(self.handles.handle, **engine_kwargs) self.handles.handle.seek(0) - self.sheets = {name: self.book[name] for name in self.book.sheetnames} - else: # Create workbook object with default optimized_write=True. self.book = Workbook(**engine_kwargs) @@ -77,6 +75,10 @@ def __init__( if self.book.worksheets: self.book.remove(self.book.worksheets[0]) + @property + def sheets(self) -> dict[str, Any]: + return {name: self.book[name] for name in self.book.sheetnames} + def save(self) -> None: """ Save workbook to disk. @@ -440,7 +442,6 @@ def write_cells( target_index = self.book.index(old_wks) del self.book[sheet_name] wks = self.book.create_sheet(sheet_name, target_index) - self.sheets[sheet_name] = wks elif self.if_sheet_exists == "error": raise ValueError( f"Sheet '{sheet_name}' already exists and " @@ -458,7 +459,6 @@ def write_cells( else: wks = self.book.create_sheet() wks.title = sheet_name - self.sheets[sheet_name] = wks if validate_freeze_panes(freeze_panes): freeze_panes = cast(Tuple[int, int], freeze_panes) diff --git a/pandas/io/excel/_xlsxwriter.py b/pandas/io/excel/_xlsxwriter.py index 49c87732f1429..bbdf11e345989 100644 --- a/pandas/io/excel/_xlsxwriter.py +++ b/pandas/io/excel/_xlsxwriter.py @@ -222,11 +222,9 @@ def write_cells( # Write the frame cells using xlsxwriter. sheet_name = self._get_sheet_name(sheet_name) - if sheet_name in self.sheets: - wks = self.sheets[sheet_name] - else: + wks = self.book.get_worksheet_by_name(sheet_name) + if wks is None: wks = self.book.add_worksheet(sheet_name) - self.sheets[sheet_name] = wks style_dict = {"null": None} diff --git a/pandas/io/excel/_xlwt.py b/pandas/io/excel/_xlwt.py index 1ada0eb25f81c..239eff10d81a9 100644 --- a/pandas/io/excel/_xlwt.py +++ b/pandas/io/excel/_xlwt.py @@ -63,6 +63,11 @@ def __init__( self.fm_datetime = xlwt.easyxf(num_format_str=self.datetime_format) self.fm_date = xlwt.easyxf(num_format_str=self.date_format) + @property + def sheets(self) -> dict[str, Any]: + result = {sheet.name: sheet for sheet in self.book._Workbook__worksheets} + return result + def save(self) -> None: """ Save workbook to disk. diff --git a/pandas/tests/io/excel/test_odswriter.py b/pandas/tests/io/excel/test_odswriter.py index 0e6d1dac55506..d7348d8677a79 100644 --- a/pandas/tests/io/excel/test_odswriter.py +++ b/pandas/tests/io/excel/test_odswriter.py @@ -56,3 +56,13 @@ def test_engine_kwargs(ext, engine_kwargs): else: with ExcelWriter(f, engine="odf", engine_kwargs=engine_kwargs) as _: pass + + +def test_book_and_sheets_consistent(ext): + # GH#??? - Ensure sheets is updated if user modifies book + with tm.ensure_clean(ext) as f: + writer = ExcelWriter(f) + assert writer.sheets == {} + table = odf.table.Table(name="test_name") + writer.book.spreadsheet.addElement(table) + assert writer.sheets == {"test_name": table} diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 9f6e1ed9c08d9..0689bc5fe90be 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -379,3 +379,12 @@ def test_read_empty_with_blank_row(datapath, ext, read_only): result = pd.read_excel(wb, engine="openpyxl") expected = DataFrame() tm.assert_frame_equal(result, expected) + + +def test_book_and_sheets_consistent(ext): + # GH#??? - Ensure sheets is updated if user modifies book + with tm.ensure_clean(ext) as f: + writer = ExcelWriter(f, engine="openpyxl") + assert writer.sheets == {} + sheet = writer.book.create_sheet("test_name", 0) + assert writer.sheets == {"test_name": sheet} diff --git a/pandas/tests/io/excel/test_xlwt.py b/pandas/tests/io/excel/test_xlwt.py index ec333defd85ac..e22616952d69d 100644 --- a/pandas/tests/io/excel/test_xlwt.py +++ b/pandas/tests/io/excel/test_xlwt.py @@ -125,3 +125,12 @@ def test_engine_kwargs(ext, style_compression): assert writer.book._Workbook__styles.style_compression == style_compression # xlwt won't allow us to close without writing something DataFrame().to_excel(writer) + + +def test_book_and_sheets_consistent(ext): + # GH#??? - Ensure sheets is updated if user modifies book + with tm.ensure_clean(ext) as f: + writer = ExcelWriter(f) + assert writer.sheets == {} + sheet = writer.book.add_sheet("test_name") + assert writer.sheets == {"test_name": sheet} From ba7ff3567d6ff5abb5add27c1e9f8946770631d8 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Fri, 28 Jan 2022 17:28:17 -0500 Subject: [PATCH 2/6] Add GH # in tests --- pandas/tests/io/excel/test_odswriter.py | 2 +- pandas/tests/io/excel/test_openpyxl.py | 2 +- pandas/tests/io/excel/test_xlwt.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/excel/test_odswriter.py b/pandas/tests/io/excel/test_odswriter.py index d7348d8677a79..c52a7c9e9caca 100644 --- a/pandas/tests/io/excel/test_odswriter.py +++ b/pandas/tests/io/excel/test_odswriter.py @@ -59,7 +59,7 @@ def test_engine_kwargs(ext, engine_kwargs): def test_book_and_sheets_consistent(ext): - # GH#??? - Ensure sheets is updated if user modifies book + # GH#45687 - Ensure sheets is updated if user modifies book with tm.ensure_clean(ext) as f: writer = ExcelWriter(f) assert writer.sheets == {} diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index 0689bc5fe90be..a5bacaa593ad7 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -382,7 +382,7 @@ def test_read_empty_with_blank_row(datapath, ext, read_only): def test_book_and_sheets_consistent(ext): - # GH#??? - Ensure sheets is updated if user modifies book + # GH#45687 - Ensure sheets is updated if user modifies book with tm.ensure_clean(ext) as f: writer = ExcelWriter(f, engine="openpyxl") assert writer.sheets == {} diff --git a/pandas/tests/io/excel/test_xlwt.py b/pandas/tests/io/excel/test_xlwt.py index e22616952d69d..3827367348e41 100644 --- a/pandas/tests/io/excel/test_xlwt.py +++ b/pandas/tests/io/excel/test_xlwt.py @@ -128,7 +128,7 @@ def test_engine_kwargs(ext, style_compression): def test_book_and_sheets_consistent(ext): - # GH#??? - Ensure sheets is updated if user modifies book + # GH#45687 - Ensure sheets is updated if user modifies book with tm.ensure_clean(ext) as f: writer = ExcelWriter(f) assert writer.sheets == {} From 8f433f02eb1f05294d89459e6cf91144bca70d16 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sat, 29 Jan 2022 07:20:37 -0500 Subject: [PATCH 3/6] Use context manager in tests --- pandas/tests/io/excel/test_odswriter.py | 10 +++++----- pandas/tests/io/excel/test_openpyxl.py | 8 ++++---- pandas/tests/io/excel/test_xlwt.py | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pandas/tests/io/excel/test_odswriter.py b/pandas/tests/io/excel/test_odswriter.py index c52a7c9e9caca..e9dad0c7fedc9 100644 --- a/pandas/tests/io/excel/test_odswriter.py +++ b/pandas/tests/io/excel/test_odswriter.py @@ -61,8 +61,8 @@ def test_engine_kwargs(ext, engine_kwargs): def test_book_and_sheets_consistent(ext): # GH#45687 - Ensure sheets is updated if user modifies book with tm.ensure_clean(ext) as f: - writer = ExcelWriter(f) - assert writer.sheets == {} - table = odf.table.Table(name="test_name") - writer.book.spreadsheet.addElement(table) - assert writer.sheets == {"test_name": table} + with ExcelWriter(f) as writer: + assert writer.sheets == {} + table = odf.table.Table(name="test_name") + writer.book.spreadsheet.addElement(table) + assert writer.sheets == {"test_name": table} diff --git a/pandas/tests/io/excel/test_openpyxl.py b/pandas/tests/io/excel/test_openpyxl.py index a5bacaa593ad7..e1c68eeaee356 100644 --- a/pandas/tests/io/excel/test_openpyxl.py +++ b/pandas/tests/io/excel/test_openpyxl.py @@ -384,7 +384,7 @@ def test_read_empty_with_blank_row(datapath, ext, read_only): def test_book_and_sheets_consistent(ext): # GH#45687 - Ensure sheets is updated if user modifies book with tm.ensure_clean(ext) as f: - writer = ExcelWriter(f, engine="openpyxl") - assert writer.sheets == {} - sheet = writer.book.create_sheet("test_name", 0) - assert writer.sheets == {"test_name": sheet} + with ExcelWriter(f, engine="openpyxl") as writer: + assert writer.sheets == {} + sheet = writer.book.create_sheet("test_name", 0) + assert writer.sheets == {"test_name": sheet} diff --git a/pandas/tests/io/excel/test_xlwt.py b/pandas/tests/io/excel/test_xlwt.py index 3827367348e41..2d5386d6c616d 100644 --- a/pandas/tests/io/excel/test_xlwt.py +++ b/pandas/tests/io/excel/test_xlwt.py @@ -130,7 +130,7 @@ def test_engine_kwargs(ext, style_compression): def test_book_and_sheets_consistent(ext): # GH#45687 - Ensure sheets is updated if user modifies book with tm.ensure_clean(ext) as f: - writer = ExcelWriter(f) - assert writer.sheets == {} - sheet = writer.book.add_sheet("test_name") - assert writer.sheets == {"test_name": sheet} + with ExcelWriter(f) as writer: + assert writer.sheets == {} + sheet = writer.book.add_sheet("test_name") + assert writer.sheets == {"test_name": sheet} From 7ede1a85528dec4e22acd1490a224875b61a70b6 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sun, 30 Jan 2022 11:41:45 -0500 Subject: [PATCH 4/6] Add back xlsxwriter sheets, add tests --- pandas/io/excel/_openpyxl.py | 3 ++- pandas/io/excel/_xlsxwriter.py | 5 +++++ pandas/tests/io/excel/test_xlsxwriter.py | 9 +++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index c91ee6398c57d..475f78b012373 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -77,7 +77,8 @@ def __init__( @property def sheets(self) -> dict[str, Any]: - return {name: self.book[name] for name in self.book.sheetnames} + result = {name: self.book[name] for name in self.book.sheetnames} + return result def save(self) -> None: """ diff --git a/pandas/io/excel/_xlsxwriter.py b/pandas/io/excel/_xlsxwriter.py index bbdf11e345989..dbd6264827591 100644 --- a/pandas/io/excel/_xlsxwriter.py +++ b/pandas/io/excel/_xlsxwriter.py @@ -205,6 +205,11 @@ def __init__( self.book = Workbook(self.handles.handle, **engine_kwargs) + @property + def sheets(self) -> dict[str, Any]: + result = self.book.sheetnames + return result + def save(self) -> None: """ Save workbook to disk. diff --git a/pandas/tests/io/excel/test_xlsxwriter.py b/pandas/tests/io/excel/test_xlsxwriter.py index b5c1b47775089..82d47a13aefbc 100644 --- a/pandas/tests/io/excel/test_xlsxwriter.py +++ b/pandas/tests/io/excel/test_xlsxwriter.py @@ -83,3 +83,12 @@ def test_engine_kwargs(ext, nan_inf_to_errors): with tm.ensure_clean(ext) as f: with ExcelWriter(f, engine="xlsxwriter", engine_kwargs=engine_kwargs) as writer: assert writer.book.nan_inf_to_errors == nan_inf_to_errors + + +def test_book_and_sheets_consistent(ext): + # GH#45687 - Ensure sheets is updated if user modifies book + with tm.ensure_clean(ext) as f: + with ExcelWriter(f, engine="xlsxwriter") as writer: + assert writer.sheets == {} + sheet = writer.book.add_worksheet("test_name") + assert writer.sheets == {"test_name": sheet} From 61c92c2fe78ad627008e76927a5eca65b801f68b Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sun, 30 Jan 2022 11:53:59 -0500 Subject: [PATCH 5/6] Add sheets as an abstract property --- pandas/io/excel/_base.py | 6 ++++++ pandas/tests/io/excel/test_writers.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index f22ff4b55eadb..ea459f13dfdc0 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -1048,6 +1048,12 @@ def engine(self) -> str: """Name of engine.""" pass + @property + @abc.abstractmethod + def sheets(self) -> dict[str, Any]: + """Mapping of sheet name to sheet object.""" + pass + @abc.abstractmethod def write_cells( self, diff --git a/pandas/tests/io/excel/test_writers.py b/pandas/tests/io/excel/test_writers.py index 6f06ef9c09e52..7a95dcb4e3a72 100644 --- a/pandas/tests/io/excel/test_writers.py +++ b/pandas/tests/io/excel/test_writers.py @@ -1271,10 +1271,12 @@ def test_register_writer(self): # some awkward mocking to test out dispatch and such actually works called_save = [] called_write_cells = [] + called_sheets = [] class DummyClass(ExcelWriter): called_save = False called_write_cells = False + called_sheets = False supported_extensions = ["xlsx", "xls"] engine = "dummy" @@ -1284,12 +1286,18 @@ def save(self): def write_cells(self, *args, **kwargs): called_write_cells.append(True) + @property + def sheets(self): + called_sheets.append(True) + def check_called(func): func() assert len(called_save) >= 1 assert len(called_write_cells) >= 1 + assert len(called_sheets) == 0 del called_save[:] del called_write_cells[:] + del called_sheets[:] with option_context("io.excel.xlsx.writer", "dummy"): path = "something.xlsx" From d8415bfa4f32d570590a8595cbad8df98dde2f24 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Mon, 31 Jan 2022 16:03:00 -0500 Subject: [PATCH 6/6] docstrings --- pandas/io/excel/_base.py | 2 +- pandas/io/excel/_odswriter.py | 1 + pandas/io/excel/_openpyxl.py | 1 + pandas/io/excel/_xlwt.py | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index ea459f13dfdc0..b33955737a111 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -1051,7 +1051,7 @@ def engine(self) -> str: @property @abc.abstractmethod def sheets(self) -> dict[str, Any]: - """Mapping of sheet name to sheet object.""" + """Mapping of sheet names to sheet objects.""" pass @abc.abstractmethod diff --git a/pandas/io/excel/_odswriter.py b/pandas/io/excel/_odswriter.py index e30548f220bbc..9069a37ccb5af 100644 --- a/pandas/io/excel/_odswriter.py +++ b/pandas/io/excel/_odswriter.py @@ -60,6 +60,7 @@ def __init__( @property def sheets(self) -> dict[str, Any]: + """Mapping of sheet names to sheet objects.""" from odf.table import Table result = { diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 475f78b012373..4b03c2536b31b 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -77,6 +77,7 @@ def __init__( @property def sheets(self) -> dict[str, Any]: + """Mapping of sheet names to sheet objects.""" result = {name: self.book[name] for name in self.book.sheetnames} return result diff --git a/pandas/io/excel/_xlwt.py b/pandas/io/excel/_xlwt.py index 239eff10d81a9..fe2addc890c22 100644 --- a/pandas/io/excel/_xlwt.py +++ b/pandas/io/excel/_xlwt.py @@ -65,6 +65,7 @@ def __init__( @property def sheets(self) -> dict[str, Any]: + """Mapping of sheet names to sheet objects.""" result = {sheet.name: sheet for sheet in self.book._Workbook__worksheets} return result