Read from multiple <tbody> within a <table>

refs pandas-dev#20690

Author: adamhooper

pandas/io/html.py

@@ -324,7 +324,7 @@ def _parse_thead(self, table):
         raise com.AbstractMethodError(self)
 
     def _parse_tbody(self, table):
-        """Return the body of the table.
+        """Return the list of tbody elements from the parsed table element.
 
         Parameters
         ----------
@@ -333,8 +333,8 @@ def _parse_tbody(self, table):
 
         Returns
         -------
-        tbody : node-like
-            A <tbody>...</tbody> element.
+        tbodys : list of node-like
+            A list of <tbody>...</tbody> elements
         """
         raise com.AbstractMethodError(self)
 
@@ -388,13 +388,17 @@ def _parse_raw_tfoot(self, table):
             np.array(res).squeeze()) if res and len(res) == 1 else res
 
     def _parse_raw_tbody(self, table):
-        tbody = self._parse_tbody(table)
+        tbodies = self._parse_tbody(table)
 
-        try:
-            res = self._parse_tr(tbody[0])
-        except IndexError:
-            res = self._parse_tr(table)
-        return self._parse_raw_data(res)
+        raw_data = []
+
+        if len(tbodies) > 0:
+            for tbody in tbodies:
+                raw_data.extend(self._parse_tr(tbody))
+        else:
+            raw_data.extend(self._parse_tr(table))
+
+        return self._parse_raw_data(raw_data)
 
     def _handle_hidden_tables(self, tbl_list, attr_name):
         """Returns list of tables, potentially removing hidden elements

pandas/tests/io/test_html.py

@@ -396,6 +396,34 @@ def test_empty_tables(self):
         res2 = self.read_html(StringIO(data2))
         assert_framelist_equal(res1, res2)
 
+    def test_multiple_tbody(self):
+        """
+        Read all tbody tags within a single table.
+        """
+        data = '''<table>
+            <thead>
+                <tr>
+                    <th>A</th>
+                    <th>B</th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td>1</td>
+                    <td>2</td>
+                </tr>
+            </tbody>
+            <tbody>
+                <tr>
+                    <td>3</td>
+                    <td>4</td>
+                </tr>
+            </tbody>
+        </table>'''
+        expected = DataFrame({ 'A': [ 1, 3 ], 'B': [ 2, 4 ] })
+        result = self.read_html(StringIO(data))
+        tm.assert_frame_equal(result[0], expected)
+
     def test_header_and_one_column(self):
         """
         Don't fail with bs4 when there is a header and only one column