BUG: change lxml remove to drop_tree (#51629) (#52135)

* BUG: change lxml remove to drop_tree (#51629)

* for removing elements when display:none in read_html
* test added

* DOC: #51629 added to whatsnew

Author: m-ganko

doc/source/whatsnew/v2.1.0.rst

@@ -205,6 +205,7 @@ MultiIndex
 
 I/O
 ^^^
+- Bug in :func:`read_html`, tail texts were removed together with elements containing ``display:none`` style (:issue:`51629`)
 - :meth:`DataFrame.to_orc` now raising ``ValueError`` when non-default :class:`Index` is given (:issue:`51828`)
 -
 

pandas/io/html.py

@@ -732,7 +732,7 @@ def _parse_tables(self, doc, match, kwargs):
                 # attribute and iterate them to check for display:none
                 for elem in table.xpath(".//*[@style]"):
                     if "display:none" in elem.attrib.get("style", "").replace(" ", ""):
-                        elem.getparent().remove(elem)
+                        elem.drop_tree()
 
         if not tables:
             raise ValueError(f"No tables found matching regex {repr(pattern)}")

pandas/tests/io/test_html.py

@@ -1238,6 +1238,28 @@ def test_displayed_only(self, displayed_only, exp0, exp1):
         else:
             assert len(dfs) == 1  # Should not parse hidden table
 
+    @pytest.mark.parametrize("displayed_only", [True, False])
+    def test_displayed_only_with_many_elements(self, displayed_only):
+        html_table = """
+        <table>
+            <tr>
+                <th>A</th>
+                <th>B</th>
+            </tr>
+            <tr>
+                <td>1</td>
+                <td>2</td>
+            </tr>
+            <tr>
+                <td><span style="display:none"></span>4</td>
+                <td>5</td>
+            </tr>
+        </table>
+        """
+        result = read_html(html_table, displayed_only=displayed_only)[0]
+        expected = DataFrame({"A": [1, 4], "B": [2, 5]})
+        tm.assert_frame_equal(result, expected)
+
     @pytest.mark.filterwarnings(
         "ignore:You provided Unicode markup but also provided a value for "
         "from_encoding.*:UserWarning"