Merge branch 'add-footer-to-read-html' of https://github.com/mjsu/pandas into mjsu-add-footer-to-read-html

Conflicts:
	doc/source/whatsnew/v0.15.2.txt

Author: jreback

doc/source/whatsnew/v0.15.2.txt

@@ -82,6 +82,7 @@ Enhancements
 - Added context manager to ``HDFStore`` for automatic closing (:issue:`8791`).
 - ``to_datetime`` gains an ``exact`` keyword to allow for a format to not require an exact match for a provided format string (if its ``False). ``exact`` defaults to ``True`` (meaning that exact matching is still the default)  (:issue:`8904`)
 - Added ``axvlines`` boolean option to parallel_coordinates plot function, determines whether vertical lines will be printed, default is True
+- Added ability to read table footers to read_html (:issue:`8552`)
 
 .. _whatsnew_0152.performance:
 

pandas/io/html.py

@@ -577,7 +577,7 @@ def _parse_raw_thead(self, table):
                 table.xpath(expr)]
 
     def _parse_raw_tfoot(self, table):
-        expr = './/tfoot//th'
+        expr = './/tfoot//th|//tfoot//td'
         return [_remove_whitespace(x.text_content()) for x in
                 table.xpath(expr)]
 
@@ -594,14 +594,17 @@ def _expand_elements(body):
 
 def _data_to_frame(data, header, index_col, skiprows, infer_types,
                    parse_dates, tupleize_cols, thousands):
-    head, body, _ = data  # _ is footer which is rarely used: ignore for now
+    head, body, foot = data
 
     if head:
         body = [head] + body
 
         if header is None:  # special case when a table has <th> elements
             header = 0
 
+    if foot:
+        body += [foot]
+
     # fill out elements of body that are "ragged"
     _expand_elements(body)
 

pandas/io/tests/test_html.py

@@ -426,6 +426,38 @@ def test_empty_tables(self):
         res1 = self.read_html(StringIO(data1))
         res2 = self.read_html(StringIO(data2))
         assert_framelist_equal(res1, res2)
+    
+    def test_tfoot_read(self):
+        """
+        Make sure that read_html reads tfoot, containing td or th. 
+        Ignores empty tfoot
+        """
+        data_template = '''<table>
+            <thead>
+                <tr>
+                    <th>A</th>
+                    <th>B</th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td>bodyA</td>
+                    <td>bodyB</td>
+                </tr>
+            </tbody>
+            <tfoot>
+                {footer}
+            </tfoot>
+        </table>'''
+
+        data1 = data_template.format(footer = "")
+        data2 = data_template.format(footer ="<tr><td>footA</td><th>footB</th></tr>")
+    
+        d1 = {'A': ['bodyA'], 'B': ['bodyB']}
+        d2 = {'A': ['bodyA', 'footA'], 'B': ['bodyB', 'footB']}
+    
+        tm.assert_frame_equal(self.read_html(data1)[0], DataFrame(d1))
+        tm.assert_frame_equal(self.read_html(data2)[0], DataFrame(d2))
 
     def test_countries_municipalities(self):
         # GH5048