ENH: Optimize nrows in read_excel

doc/source/whatsnew/v1.1.0.rst

@@ -690,6 +690,7 @@ Performance improvements
 - Performance improvement in arithmetic operations between two :class:`DataFrame` objects (:issue:`32779`)
 - Performance improvement in :class:`pandas.core.groupby.RollingGroupby` (:issue:`34052`)
 - Performance improvement in arithmetic operations (sub, add, mul, div) for MultiIndex (:issue:`34297`)
+- Performance improvement in `read_excel` for integer ``header``, ``skiprows``, and ``nrows`` (:issue:`33281`).
 
 .. ---------------------------------------------------------------------------
 

pandas/core/sorting.py

@@ -385,7 +385,7 @@ def ensure_key_mapped(values, key: Optional[Callable], levels=None):
     from pandas.core.indexes.api import Index
 
     if not key:
-        return values.copy()
+        return values
 
     if isinstance(values, ABCMultiIndex):
         return ensure_key_mapped_multiindex(values, key, level=levels)

pandas/io/excel/_base.py

@@ -3,10 +3,12 @@
 from io import BytesIO
 import os
 from textwrap import fill
+from typing import List, Optional, Sequence, Tuple
 
 from pandas._config import config
 
 from pandas._libs.parsers import STR_NA_VALUES
+from pandas._typing import Scalar, Union
 from pandas.errors import EmptyDataError
 from pandas.util._decorators import Appender
 
@@ -383,9 +385,50 @@ def get_sheet_by_index(self, index):
         pass
 
     @abc.abstractmethod
-    def get_sheet_data(self, sheet, convert_float):
+    def get_sheet_data(
+        self,
+        sheet,
+        convert_float: bool,
+        header: Optional[Union[int, Sequence[int]]],
+        skiprows: Optional[Union[int, Sequence[int]]],
+        nrows: Optional[int],
+    ) -> List[List[Scalar]]:
         pass
 
+    def should_skip_row(
+        self,
+        index: int,
+        header: Optional[Union[int, Sequence[int]]],
+        skiprows: Optional[Union[int, Sequence[int]]],
+        nrows: Optional[int],
+    ) -> Tuple[bool, bool]:
+        """
+        Determines whether row should be read.
+
+        Parameters
+        ----------
+        index : int
+            Index of row.
+        header : int, list of int
+            Rows used as column labels.
+        skiprows : int, list of int
+            Rows to skip at the begining.
+        nrows : int
+            Number of rows to parse.
+
+        Returns
+        -------
+        Tuple with the first bool element determining if row should be
+        skipped and second bool element determining if reading should be stopped.
+        """
+        if nrows is not None and isinstance(header, int) and isinstance(skiprows, int):
+            if index < header + skiprows - 1:
+                return True, False
+            if index <= header + skiprows + nrows:
+                return False, False
+            return False, True
+        return False, False
+
     def parse(
         self,
         sheet_name=0,
@@ -439,7 +482,16 @@ def parse(
             else:  # assume an integer if not a string
                 sheet = self.get_sheet_by_index(asheetname)
 
-            data = self.get_sheet_data(sheet, convert_float)
+            gsd_header = 0 if header is None else header
+            gsd_skiprows = 0 if skiprows is None else skiprows
+            gsd_nrows = nrows if isinstance(nrows, int) else None
+
+            if isinstance(gsd_header, list) or isinstance(gsd_skiprows, list):
+                gsd_nrows = None
+
+            data = self.get_sheet_data(
+                sheet, convert_float, gsd_header, gsd_skiprows, gsd_nrows
+            )
             usecols = _maybe_convert_usecols(usecols)
 
             if not data:

pandas/io/excel/_odfreader.py

@@ -1,6 +1,6 @@
-from typing import List
+from typing import List, Optional, Sequence
 
-from pandas._typing import FilePathOrBuffer, Scalar
+from pandas._typing import FilePathOrBuffer, Scalar, Union
 from pandas.compat._optional import import_optional_dependency
 
 import pandas as pd
@@ -63,7 +63,14 @@ def get_sheet_by_name(self, name: str):
 
         raise ValueError(f"sheet {name} not found")
 
-    def get_sheet_data(self, sheet, convert_float: bool) -> List[List[Scalar]]:
+    def get_sheet_data(
+        self,
+        sheet,
+        convert_float: bool,
+        header: Optional[Union[int, Sequence[int]]],
+        skiprows: Optional[Union[int, Sequence[int]]],
+        nrows: Optional[int],
+    ) -> List[List[Scalar]]:
         """
         Parse an ODF Table into a list of lists
         """
@@ -80,6 +87,16 @@ def get_sheet_data(self, sheet, convert_float: bool) -> List[List[Scalar]]:
         table: List[List[Scalar]] = []
 
         for i, sheet_row in enumerate(sheet_rows):
+
+            should_continue, should_break = self.should_skip_row(
+                i, header, skiprows, nrows
+            )
+            if should_continue:
+                table.append([])
+                continue
+            if should_break:
+                break
+
             sheet_cells = [x for x in sheet_row.childNodes if x.qname in cell_names]
             empty_cells = 0
             table_row: List[Scalar] = []

pandas/io/excel/_openpyxl.py

@@ -1,8 +1,8 @@
-from typing import List
+from typing import List, Optional, Sequence
 
 import numpy as np
 
-from pandas._typing import FilePathOrBuffer, Scalar
+from pandas._typing import FilePathOrBuffer, Scalar, Union
 from pandas.compat._optional import import_optional_dependency
 
 from pandas.io.excel._base import ExcelWriter, _BaseExcelReader
@@ -529,9 +529,18 @@ def _convert_cell(self, cell, convert_float: bool) -> Scalar:
 
         return cell.value
 
-    def get_sheet_data(self, sheet, convert_float: bool) -> List[List[Scalar]]:
+    def get_sheet_data(
+        self,
+        sheet,
+        convert_float: bool,
+        header: Optional[Union[int, Sequence[int]]],
+        skiprows: Optional[Union[int, Sequence[int]]],
+        nrows: Optional[int],
+    ) -> List[List[Scalar]]:
         data: List[List[Scalar]] = []
+
         for row in sheet.rows:
+
             data.append([self._convert_cell(cell, convert_float) for cell in row])
 
         return data

pandas/io/excel/_pyxlsb.py

@@ -1,6 +1,6 @@
-from typing import List
+from typing import List, Optional, Sequence
 
-from pandas._typing import FilePathOrBuffer, Scalar
+from pandas._typing import FilePathOrBuffer, Scalar, Union
 from pandas.compat._optional import import_optional_dependency
 
 from pandas.io.excel._base import _BaseExcelReader
@@ -62,7 +62,14 @@ def _convert_cell(self, cell, convert_float: bool) -> Scalar:
 
         return cell.v
 
-    def get_sheet_data(self, sheet, convert_float: bool) -> List[List[Scalar]]:
+    def get_sheet_data(
+        self,
+        sheet,
+        convert_float: bool,
+        header: Optional[Union[int, Sequence[int]]],
+        skiprows: Optional[Union[int, Sequence[int]]],
+        nrows: Optional[int],
+    ) -> List[List[Scalar]]:
         return [
             [self._convert_cell(c, convert_float) for c in r]
             for r in sheet.rows(sparse=False)

pandas/io/excel/_xlrd.py

@@ -1,7 +1,9 @@
 from datetime import time
+from typing import List, Optional, Sequence
 
 import numpy as np
 
+from pandas._typing import Scalar, Union
 from pandas.compat._optional import import_optional_dependency
 
 from pandas.io.excel._base import _BaseExcelReader
@@ -46,7 +48,14 @@ def get_sheet_by_name(self, name):
     def get_sheet_by_index(self, index):
         return self.book.sheet_by_index(index)
 
-    def get_sheet_data(self, sheet, convert_float):
+    def get_sheet_data(
+        self,
+        sheet,
+        convert_float: bool,
+        header: Optional[Union[int, Sequence[int]]],
+        skiprows: Optional[Union[int, Sequence[int]]],
+        nrows: Optional[int],
+    ) -> List[List[Scalar]]:
         from xlrd import (
             xldate,
             XL_CELL_DATE,
@@ -95,9 +104,19 @@ def _parse_cell(cell_contents, cell_typ):
                     cell_contents = val
             return cell_contents
 
-        data = []
+        data: List[List[Scalar]] = []
 
         for i in range(sheet.nrows):
+
+            should_continue, should_break = self.should_skip_row(
+                i, header, skiprows, nrows
+            )
+            if should_continue:
+                data.append([])
+                continue
+            if should_break:
+                break
+
             row = [
                 _parse_cell(value, typ)
                 for value, typ in zip(sheet.row_values(i), sheet.row_types(i))

pandas/tests/io/excel/test_readers.py

@@ -1084,3 +1084,17 @@ def test_excel_high_surrogate(self, engine):
         # should not produce a segmentation violation
         actual = pd.read_excel("high_surrogate.xlsx")
         tm.assert_frame_equal(expected, actual)
+
+    @pytest.mark.parametrize("header, skiprows", [(1, 2), (0, 3)])
+    @td.check_file_leaks
+    def test_header_skiprows_nrows(self, engine, read_ext, header, skiprows):
+        data = pd.read_excel("test1" + read_ext, engine=engine)
+        expected = (
+            DataFrame(data.iloc[3:6])
+            .reset_index(drop=True)
+            .rename(columns=data.iloc[2].rename(None))
+        )
+        actual = pd.read_excel(
+            "test1" + read_ext, engine=engine, header=header, skiprows=skiprows, nrows=3
+        )
+        tm.assert_frame_equal(expected, actual)