From 590aa141b9908d0ed42a3e26380288c10cbf5964 Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Tue, 22 Jun 2021 10:57:11 +0200 Subject: [PATCH 01/10] create a cleaner copy mechanism and future proof tests --- pandas/io/formats/style.py | 70 ++++++--- pandas/tests/io/formats/style/test_style.py | 164 ++++++++------------ 2 files changed, 114 insertions(+), 120 deletions(-) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index c03275b565fd4..257a754725cb8 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -964,32 +964,54 @@ def _update_ctx(self, attrs: DataFrame) -> None: self.ctx[(i, j)].extend(css_list) def _copy(self, deepcopy: bool = False) -> Styler: - styler = Styler( - self.data, - precision=self.precision, - caption=self.caption, - table_attributes=self.table_attributes, - cell_ids=self.cell_ids, - na_rep=self.na_rep, - ) + """ + Copies a Styler, allowing for deepcopy or shallow copy - styler.uuid = self.uuid - styler.hide_index_ = self.hide_index_ + Copying a Styler aims to recreate a new Styler object which contains the same + data and styles as the original. - if deepcopy: - styler.ctx = copy.deepcopy(self.ctx) - styler._todo = copy.deepcopy(self._todo) - styler.table_styles = copy.deepcopy(self.table_styles) - styler.hidden_columns = copy.copy(self.hidden_columns) - styler.cell_context = copy.deepcopy(self.cell_context) - styler.tooltips = copy.deepcopy(self.tooltips) - else: - styler.ctx = self.ctx - styler._todo = self._todo - styler.table_styles = self.table_styles - styler.hidden_columns = self.hidden_columns - styler.cell_context = self.cell_context - styler.tooltips = self.tooltips + Data dependent attributes [copied and NOT exported]: + - formatting (._display_funcs) + - hidden index values or column values (.hidden_rows, .hidden_columns) + - tooltips + - cell_context (cell css classes) + - ctx (cell css styles) + - applied styles (_todo) + - caption + + Non-data dependent attributes [copied and exported]: + - hidden index state and hidden columns state (.hide_index_, .hide_columns_) + - table_attributes + - table_styles + + """ + styler = Styler( + self.data, # populates attributes 'data', 'columns', 'index' as shallow + uuid_len=self.uuid_len, + ) + shallow = [ # simple string or boolean immutables + "hide_index_", + "hide_columns_", + "table_attributes", + "cell_ids", + ] + deep = [ # nested lists or dicts + "_display_funcs", + "hidden_rows", + "hidden_columns", + "ctx", + "cell_context", + "_todo", + "table_styles", + "tooltips", + ] + + for attr in shallow: + setattr(styler, attr, getattr(self, attr)) + + op = copy.deepcopy if deepcopy else lambda x: x + for attr in deep: + setattr(styler, attr, op(getattr(self, attr))) return styler diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py index 0516aa6029487..cbf52bca0ce19 100644 --- a/pandas/tests/io/formats/style/test_style.py +++ b/pandas/tests/io/formats/style/test_style.py @@ -38,6 +38,32 @@ def mi_styler(mi_df): return Styler(mi_df, uuid_len=0) +@pytest.fixture +def mi_styler_comp(mi_styler): + # comprehensively add features to mi_styler + mi_styler.set_table_styles([{"selector": "a", "props": "a:v;"}]) + mi_styler.hide_columns() + mi_styler.hide_columns([("c0", "c1_a")]) + mi_styler.hide_index() + mi_styler.hide_index([("i0", "i1_a")]) + mi_styler.set_table_attributes('class="box"') + mi_styler.format(na_rep="MISSING", precision=3) + mi_styler.highlight_max(axis=None) + mi_styler.set_td_classes( + DataFrame( + [["a", "b"], ["a", "c"]], index=mi_styler.index, columns=mi_styler.columns + ) + ) + mi_styler.set_tooltips( + DataFrame( + [["a2", "b2"], ["a2", "c2"]], + index=mi_styler.index, + columns=mi_styler.columns, + ) + ) + return mi_styler + + @pytest.mark.parametrize( "sparse_columns, exp_cols", [ @@ -156,6 +182,48 @@ def test_render_trimming_mi(): assert {"attributes": 'colspan="2"'}.items() <= ctx["head"][0][2].items() +@pytest.mark.parametrize("comp", [True, False]) +@pytest.mark.parametrize("render", [True, False]) +@pytest.mark.parametrize("deepcopy", [True, False]) +def test_copy(comp, render, deepcopy, mi_styler, mi_styler_comp): + styler = mi_styler_comp if comp else mi_styler + styler.uuid_len = 5 + + s2 = copy.deepcopy(styler) if deepcopy else copy.copy(styler) # make copy and check + assert s2 is not styler + + if render: + styler.to_html() + + excl = ["na_rep", "precision", "uuid", "cellstyle_map"] # deprecated or special var + if not deepcopy: # check memory locations are equal for all included attributes + for attr in [a for a in styler.__dict__ if (not callable(a) and a not in excl)]: + assert id(getattr(s2, attr)) == id(getattr(styler, attr)) + else: # check memory locations are different for nested or mutable vars + shallow = [ + "data", + "columns", + "index", + "uuid_len", + "cell_ids", + "hide_index_", + "hide_columns_", + "table_attributes", + ] + for attr in shallow: + assert id(getattr(s2, attr)) == id(getattr(styler, attr)) + + for attr in [ + a + for a in styler.__dict__ + if (not callable(a) and a not in excl and a not in shallow) + ]: + if getattr(s2, attr) is None: + assert id(getattr(s2, attr)) == id(getattr(styler, attr)) + else: + assert id(getattr(s2, attr)) != id(getattr(styler, attr)) + + class TestStyler: def setup_method(self, method): np.random.seed(24) @@ -211,102 +279,6 @@ def test_update_ctx_flatten_multi_and_trailing_semi(self): } assert self.styler.ctx == expected - @pytest.mark.parametrize("do_changes", [True, False]) - @pytest.mark.parametrize("do_render", [True, False]) - def test_copy(self, do_changes, do_render): - # Updated in GH39708 - # Change some defaults (to check later if the new values are copied) - if do_changes: - self.styler.set_table_styles( - [{"selector": "th", "props": [("foo", "bar")]}] - ) - self.styler.set_table_attributes('class="foo" data-bar') - self.styler.hide_index_ = not self.styler.hide_index_ - self.styler.hide_columns("A") - classes = DataFrame( - [["favorite-val red", ""], [None, "blue my-val"]], - index=self.df.index, - columns=self.df.columns, - ) - self.styler.set_td_classes(classes) - ttips = DataFrame( - data=[["Favorite", ""], [np.nan, "my"]], - columns=self.df.columns, - index=self.df.index, - ) - self.styler.set_tooltips(ttips) - self.styler.cell_ids = not self.styler.cell_ids - - if do_render: - self.styler.render() - - s_copy = copy.copy(self.styler) - s_deepcopy = copy.deepcopy(self.styler) - - assert self.styler is not s_copy - assert self.styler is not s_deepcopy - - # Check for identity - assert self.styler.ctx is s_copy.ctx - assert self.styler._todo is s_copy._todo - assert self.styler.table_styles is s_copy.table_styles - assert self.styler.hidden_columns is s_copy.hidden_columns - assert self.styler.cell_context is s_copy.cell_context - assert self.styler.tooltips is s_copy.tooltips - if do_changes: # self.styler.tooltips is not None - assert self.styler.tooltips.tt_data is s_copy.tooltips.tt_data - assert ( - self.styler.tooltips.class_properties - is s_copy.tooltips.class_properties - ) - assert self.styler.tooltips.table_styles is s_copy.tooltips.table_styles - - # Check for non-identity - assert self.styler.ctx is not s_deepcopy.ctx - assert self.styler._todo is not s_deepcopy._todo - assert self.styler.hidden_columns is not s_deepcopy.hidden_columns - assert self.styler.cell_context is not s_deepcopy.cell_context - if do_changes: # self.styler.table_style is not None - assert self.styler.table_styles is not s_deepcopy.table_styles - if do_changes: # self.styler.tooltips is not None - assert self.styler.tooltips is not s_deepcopy.tooltips - assert self.styler.tooltips.tt_data is not s_deepcopy.tooltips.tt_data - assert ( - self.styler.tooltips.class_properties - is not s_deepcopy.tooltips.class_properties - ) - assert ( - self.styler.tooltips.table_styles - is not s_deepcopy.tooltips.table_styles - ) - - self.styler._update_ctx(self.attrs) - self.styler.highlight_max() - assert self.styler.ctx == s_copy.ctx - assert self.styler.ctx != s_deepcopy.ctx - assert self.styler._todo == s_copy._todo - assert self.styler._todo != s_deepcopy._todo - assert s_deepcopy._todo == [] - - equal_attributes = [ - "table_styles", - "table_attributes", - "cell_ids", - "hide_index_", - "hidden_columns", - "cell_context", - ] - for s2 in [s_copy, s_deepcopy]: - for att in equal_attributes: - assert self.styler.__dict__[att] == s2.__dict__[att] - if do_changes: # self.styler.tooltips is not None - tm.assert_frame_equal(self.styler.tooltips.tt_data, s2.tooltips.tt_data) - assert ( - self.styler.tooltips.class_properties - == s2.tooltips.class_properties - ) - assert self.styler.tooltips.table_styles == s2.tooltips.table_styles - def test_clear(self): # updated in GH 39396 tt = DataFrame({"A": [None, "tt"]}) From 1bf7d0e35bf33563c1c2867a4a3d6a9a840ce97d Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Tue, 22 Jun 2021 11:16:27 +0200 Subject: [PATCH 02/10] add gh number --- pandas/io/formats/style.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 257a754725cb8..e2d2f698efc11 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -985,6 +985,7 @@ def _copy(self, deepcopy: bool = False) -> Styler: - table_styles """ + # GH 40675 styler = Styler( self.data, # populates attributes 'data', 'columns', 'index' as shallow uuid_len=self.uuid_len, @@ -1016,9 +1017,6 @@ def _copy(self, deepcopy: bool = False) -> Styler: return styler def __copy__(self) -> Styler: - """ - Deep copy by default. - """ return self._copy(deepcopy=False) def __deepcopy__(self, memo) -> Styler: From f354cb599f559d835f33fdea6155a3110dd7ac0c Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Tue, 22 Jun 2021 11:22:21 +0200 Subject: [PATCH 03/10] name chg --- pandas/tests/io/formats/style/test_style.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py index cbf52bca0ce19..65fc47c289443 100644 --- a/pandas/tests/io/formats/style/test_style.py +++ b/pandas/tests/io/formats/style/test_style.py @@ -182,11 +182,11 @@ def test_render_trimming_mi(): assert {"attributes": 'colspan="2"'}.items() <= ctx["head"][0][2].items() -@pytest.mark.parametrize("comp", [True, False]) +@pytest.mark.parametrize("comprehensive", [True, False]) @pytest.mark.parametrize("render", [True, False]) @pytest.mark.parametrize("deepcopy", [True, False]) -def test_copy(comp, render, deepcopy, mi_styler, mi_styler_comp): - styler = mi_styler_comp if comp else mi_styler +def test_copy(comprehensive, render, deepcopy, mi_styler, mi_styler_comp): + styler = mi_styler_comp if comprehensive else mi_styler styler.uuid_len = 5 s2 = copy.deepcopy(styler) if deepcopy else copy.copy(styler) # make copy and check From defc20aa3b6ee111a5c50222027f2b449975397d Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Tue, 22 Jun 2021 11:32:10 +0200 Subject: [PATCH 04/10] doc correction --- pandas/io/formats/style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index e2d2f698efc11..2288d55877581 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -976,13 +976,13 @@ def _copy(self, deepcopy: bool = False) -> Styler: - tooltips - cell_context (cell css classes) - ctx (cell css styles) - - applied styles (_todo) - caption Non-data dependent attributes [copied and exported]: - hidden index state and hidden columns state (.hide_index_, .hide_columns_) - table_attributes - table_styles + - applied styles (_todo) """ # GH 40675 From f02b20a9323ec4e8e34a4e25cfd0dd992d40e1bd Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Tue, 22 Jun 2021 11:49:21 +0200 Subject: [PATCH 05/10] add caption --- pandas/io/formats/style.py | 1 + pandas/tests/io/formats/style/test_style.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 2288d55877581..70019e586b60c 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -995,6 +995,7 @@ def _copy(self, deepcopy: bool = False) -> Styler: "hide_columns_", "table_attributes", "cell_ids", + "caption", ] deep = [ # nested lists or dicts "_display_funcs", diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py index 65fc47c289443..f2c2f673909d4 100644 --- a/pandas/tests/io/formats/style/test_style.py +++ b/pandas/tests/io/formats/style/test_style.py @@ -41,6 +41,9 @@ def mi_styler(mi_df): @pytest.fixture def mi_styler_comp(mi_styler): # comprehensively add features to mi_styler + mi_styler.uuid_len = 5 + mi_styler.uuid = "abcde_" + mi_styler.set_caption("capt") mi_styler.set_table_styles([{"selector": "a", "props": "a:v;"}]) mi_styler.hide_columns() mi_styler.hide_columns([("c0", "c1_a")]) @@ -205,6 +208,7 @@ def test_copy(comprehensive, render, deepcopy, mi_styler, mi_styler_comp): "columns", "index", "uuid_len", + "caption", "cell_ids", "hide_index_", "hide_columns_", From 193b3e8e8067f95b42132ce23cd2d25d582b4272 Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Tue, 22 Jun 2021 12:37:31 +0200 Subject: [PATCH 06/10] future proof Styler.clear for additional variables --- pandas/io/formats/style.py | 17 ++++++------- pandas/tests/io/formats/style/test_style.py | 27 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 70019e586b60c..45fa65556e131 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -1029,15 +1029,14 @@ def clear(self) -> None: Returns None. """ - self.ctx.clear() - self.tooltips = None - self.cell_context.clear() - self._todo.clear() - - self.hide_index_ = False - self.hidden_columns = [] - # self.format and self.table_styles may be dependent on user - # input in self.__init__() + # create default GH 40675 + clean_copy = Styler(self.data, uuid=self.uuid) + clean_attrs = [a for a in clean_copy.__dict__ if not callable(a)] + self_attrs = [a for a in self.__dict__ if not callable(a)] # maybe more attrs + for attr in clean_attrs: + setattr(self, attr, getattr(clean_copy, attr)) + for attr in set(self_attrs).difference(clean_attrs): + delattr(self, attr) def _apply( self, diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py index f2c2f673909d4..c19af407fd611 100644 --- a/pandas/tests/io/formats/style/test_style.py +++ b/pandas/tests/io/formats/style/test_style.py @@ -228,6 +228,33 @@ def test_copy(comprehensive, render, deepcopy, mi_styler, mi_styler_comp): assert id(getattr(s2, attr)) != id(getattr(styler, attr)) +def test_clear(mi_styler_comp): + clean_copy = Styler(mi_styler_comp.data, uuid=mi_styler_comp.uuid) + + mi_styler_comp.to_html() # new attrs maybe created on render + excl = [ + "data", + "index", + "columns", + "uuid", + "uuid_len", + "cell_ids", + "cellstyle_map", # execution time only + "precision", # deprecated + "na_rep", # deprecated + ] + # tests variables are not the same before clearing, except for excluded. + for attr in [a for a in mi_styler_comp.__dict__ if not (callable(a) or a in excl)]: + res = getattr(mi_styler_comp, attr) == getattr(clean_copy, attr) + assert not (all(res) if (hasattr(res, "__iter__") and len(res) > 0) else res) + + # test variables are same after clearing + mi_styler_comp.clear() + for attr in [a for a in mi_styler_comp.__dict__ if not (callable(a))]: + res = getattr(mi_styler_comp, attr) == getattr(clean_copy, attr) + assert all(res) if hasattr(res, "__iter__") else res + + class TestStyler: def setup_method(self, method): np.random.seed(24) From d2757ee3da8a856ee7f0b4d9745968cb7ebe22df Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Tue, 22 Jun 2021 12:55:58 +0200 Subject: [PATCH 07/10] mypy fix --- pandas/io/formats/style.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 70019e586b60c..00c18ac261bab 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -1011,9 +1011,9 @@ def _copy(self, deepcopy: bool = False) -> Styler: for attr in shallow: setattr(styler, attr, getattr(self, attr)) - op = copy.deepcopy if deepcopy else lambda x: x for attr in deep: - setattr(styler, attr, op(getattr(self, attr))) + val = getattr(self, attr) + setattr(styler, attr, copy.deepcopy(val) if deepcopy else val) return styler From 8c40d6d08e464eb05ee64c65964da426774a13c8 Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Sun, 27 Jun 2021 23:38:52 +0200 Subject: [PATCH 08/10] Merge remote-tracking branch 'upstream/master' into styler_clear_clean --- pandas/tests/io/formats/style/test_style.py | 47 +++++---------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py index c19af407fd611..aa10794a09c6e 100644 --- a/pandas/tests/io/formats/style/test_style.py +++ b/pandas/tests/io/formats/style/test_style.py @@ -229,9 +229,11 @@ def test_copy(comprehensive, render, deepcopy, mi_styler, mi_styler_comp): def test_clear(mi_styler_comp): - clean_copy = Styler(mi_styler_comp.data, uuid=mi_styler_comp.uuid) + styler = mi_styler_comp + styler.to_html() # new attrs maybe created on render + + clean_copy = Styler(styler.data, uuid=styler.uuid) - mi_styler_comp.to_html() # new attrs maybe created on render excl = [ "data", "index", @@ -243,15 +245,15 @@ def test_clear(mi_styler_comp): "precision", # deprecated "na_rep", # deprecated ] - # tests variables are not the same before clearing, except for excluded. - for attr in [a for a in mi_styler_comp.__dict__ if not (callable(a) or a in excl)]: - res = getattr(mi_styler_comp, attr) == getattr(clean_copy, attr) + # tests vars are not same vals on obj and clean copy before clear (except for excl) + for attr in [a for a in styler.__dict__ if not (callable(a) or a in excl)]: + res = getattr(styler, attr) == getattr(clean_copy, attr) assert not (all(res) if (hasattr(res, "__iter__") and len(res) > 0) else res) - # test variables are same after clearing - mi_styler_comp.clear() - for attr in [a for a in mi_styler_comp.__dict__ if not (callable(a))]: - res = getattr(mi_styler_comp, attr) == getattr(clean_copy, attr) + # test vars have same vales on obj and clean copy after clearing + styler.clear() + for attr in [a for a in styler.__dict__ if not (callable(a))]: + res = getattr(styler, attr) == getattr(clean_copy, attr) assert all(res) if hasattr(res, "__iter__") else res @@ -310,33 +312,6 @@ def test_update_ctx_flatten_multi_and_trailing_semi(self): } assert self.styler.ctx == expected - def test_clear(self): - # updated in GH 39396 - tt = DataFrame({"A": [None, "tt"]}) - css = DataFrame({"A": [None, "cls-a"]}) - s = self.df.style.highlight_max().set_tooltips(tt).set_td_classes(css) - s = s.hide_index().hide_columns("A") - # _todo, tooltips and cell_context items added to.. - assert len(s._todo) > 0 - assert s.tooltips - assert len(s.cell_context) > 0 - assert s.hide_index_ is True - assert len(s.hidden_columns) > 0 - - s = s._compute() - # ctx item affected when a render takes place. _todo is maintained - assert len(s.ctx) > 0 - assert len(s._todo) > 0 - - s.clear() - # ctx, _todo, tooltips and cell_context items all revert to null state. - assert len(s.ctx) == 0 - assert len(s._todo) == 0 - assert not s.tooltips - assert len(s.cell_context) == 0 - assert s.hide_index_ is False - assert len(s.hidden_columns) == 0 - def test_render(self): df = DataFrame({"A": [0, 1]}) style = lambda x: pd.Series(["color: red", "color: blue"], name=x.name) From 7cdc3d7c9b90a6bd6fa1caa3f7d91b27577a43b3 Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Sun, 27 Jun 2021 23:48:39 +0200 Subject: [PATCH 09/10] dev note --- pandas/tests/io/formats/style/test_style.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py index aa10794a09c6e..bda42b3626a75 100644 --- a/pandas/tests/io/formats/style/test_style.py +++ b/pandas/tests/io/formats/style/test_style.py @@ -229,6 +229,8 @@ def test_copy(comprehensive, render, deepcopy, mi_styler, mi_styler_comp): def test_clear(mi_styler_comp): + # NOTE: if this test fails for new features then 'mi_styler_comp' should be updated + # to ensure proper testing of the 'copy', 'clear', 'export' methods with new feature styler = mi_styler_comp styler.to_html() # new attrs maybe created on render From 648f526920b8696ea2291cc0ec9afdda10445421 Mon Sep 17 00:00:00 2001 From: "JHM Darbyshire (iMac)" Date: Sun, 27 Jun 2021 23:50:56 +0200 Subject: [PATCH 10/10] dev note --- pandas/tests/io/formats/style/test_style.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/formats/style/test_style.py b/pandas/tests/io/formats/style/test_style.py index bda42b3626a75..480356de2450f 100644 --- a/pandas/tests/io/formats/style/test_style.py +++ b/pandas/tests/io/formats/style/test_style.py @@ -231,6 +231,7 @@ def test_copy(comprehensive, render, deepcopy, mi_styler, mi_styler_comp): def test_clear(mi_styler_comp): # NOTE: if this test fails for new features then 'mi_styler_comp' should be updated # to ensure proper testing of the 'copy', 'clear', 'export' methods with new feature + # GH 40675 styler = mi_styler_comp styler.to_html() # new attrs maybe created on render