Moto server (#35655)

Author: martindurant

ci/deps/azure-37-locale.yaml

@@ -21,6 +21,7 @@ dependencies:
   - lxml
   - matplotlib>=3.3.0
   - moto
+  - flask
   - nomkl
   - numexpr
   - numpy=1.16.*

ci/deps/azure-37-slow.yaml

@@ -27,9 +27,11 @@ dependencies:
   - python-dateutil
   - pytz
   - s3fs>=0.4.0
+  - moto>=1.3.14
   - scipy
   - sqlalchemy
   - xlrd
   - xlsxwriter
   - xlwt
   - moto
+  - flask

ci/deps/azure-38-locale.yaml

@@ -14,6 +14,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
+  - flask
   - html5lib
   - ipython
   - jinja2
@@ -32,6 +33,7 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
+  - moto
   - pyarrow>=0.15
   - pip
   - pip:

ci/deps/azure-windows-37.yaml

@@ -15,21 +15,22 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
-  - fsspec>=0.7.4
+  - fsspec>=0.8.0
   - gcsfs>=0.6.0
   - html5lib
   - jinja2
   - lxml
   - matplotlib=2.2.*
-  - moto
+  - moto>=1.3.14
+  - flask
   - numexpr
   - numpy=1.16.*
   - openpyxl
   - pyarrow=0.15
   - pytables
   - python-dateutil
   - pytz
-  - s3fs>=0.4.0
+  - s3fs>=0.4.2
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/azure-windows-38.yaml

@@ -16,7 +16,10 @@ dependencies:
   - blosc
   - bottleneck
   - fastparquet>=0.3.2
+  - flask
+  - fsspec>=0.8.0
   - matplotlib=3.1.3
+  - moto>=1.3.14
   - numba
   - numexpr
   - numpy=1.18.*
@@ -26,6 +29,7 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
+  - s3fs>=0.4.0
   - scipy
   - xlrd
   - xlsxwriter

ci/deps/travis-37-arm64.yaml

@@ -17,5 +17,6 @@ dependencies:
   - python-dateutil
   - pytz
   - pip
+  - flask
   - pip:
     - moto

ci/deps/travis-37-cov.yaml

@@ -23,7 +23,8 @@ dependencies:
   - geopandas
   - html5lib
   - matplotlib
-  - moto
+  - moto>=1.3.14
+  - flask
   - nomkl
   - numexpr
   - numpy=1.16.*

ci/deps/travis-37-locale.yaml

@@ -21,12 +21,12 @@ dependencies:
   - jinja2
   - lxml=4.3.0
   - matplotlib=3.0.*
-  - moto
   - nomkl
   - numexpr
   - numpy
   - openpyxl
   - pandas-gbq=0.12.0
+  - pyarrow>=0.17
   - psycopg2=2.7
   - pyarrow>=0.15.0 # GH #35813
   - pymysql=0.7.11

ci/deps/travis-37.yaml

@@ -20,8 +20,8 @@ dependencies:
   - pyarrow
   - pytz
   - s3fs>=0.4.0
+  - moto>=1.3.14
+  - flask
   - tabulate
   - pyreadstat
   - pip
-  - pip:
-    - moto

doc/source/whatsnew/v1.2.0.rst

@@ -24,6 +24,9 @@ of the individual storage backends (detailed from the fsspec docs for
 `builtin implementations`_ and linked to `external ones`_). See
 Section :ref:`io.remote`.
 
+:issue:`35655` added fsspec support (including ``storage_options``)
+for reading excel files.
+
 .. _builtin implementations: https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations
 .. _external ones: https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations
 

environment.yml

@@ -51,6 +51,7 @@ dependencies:
   - botocore>=1.11
   - hypothesis>=3.82
   - moto  # mock S3
+  - flask
   - pytest>=5.0.1
   - pytest-cov
   - pytest-xdist>=1.21

pandas/io/excel/_base.py

@@ -3,11 +3,12 @@
 from io import BufferedIOBase, BytesIO, RawIOBase
 import os
 from textwrap import fill
-from typing import Union
+from typing import Any, Mapping, Union
 
 from pandas._config import config
 
 from pandas._libs.parsers import STR_NA_VALUES
+from pandas._typing import StorageOptions
 from pandas.errors import EmptyDataError
 from pandas.util._decorators import Appender, deprecate_nonkeyword_arguments
 
@@ -199,6 +200,15 @@
     Duplicate columns will be specified as 'X', 'X.1', ...'X.N', rather than
     'X'...'X'. Passing in False will cause data to be overwritten if there
     are duplicate names in the columns.
+storage_options : StorageOptions
+    Extra options that make sense for a particular storage connection, e.g.
+    host, port, username, password, etc., if using a URL that will
+    be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
+    will be raised if providing this argument with a local path or
+    a file-like buffer. See the fsspec and backend storage implementation
+    docs for the set of allowed keys and values
+
+    .. versionadded:: 1.2.0
 
 Returns
 -------
@@ -298,10 +308,11 @@ def read_excel(
     skipfooter=0,
     convert_float=True,
     mangle_dupe_cols=True,
+    storage_options: StorageOptions = None,
 ):
 
     if not isinstance(io, ExcelFile):
-        io = ExcelFile(io, engine=engine)
+        io = ExcelFile(io, storage_options=storage_options, engine=engine)
     elif engine and engine != io.engine:
         raise ValueError(
             "Engine should not be specified when passing "
@@ -336,12 +347,14 @@ def read_excel(
 
 
 class _BaseExcelReader(metaclass=abc.ABCMeta):
-    def __init__(self, filepath_or_buffer):
+    def __init__(self, filepath_or_buffer, storage_options: StorageOptions = None):
         # If filepath_or_buffer is a url, load the data into a BytesIO
         if is_url(filepath_or_buffer):
             filepath_or_buffer = BytesIO(urlopen(filepath_or_buffer).read())
         elif not isinstance(filepath_or_buffer, (ExcelFile, self._workbook_class)):
-            filepath_or_buffer, _, _, _ = get_filepath_or_buffer(filepath_or_buffer)
+            filepath_or_buffer, _, _, _ = get_filepath_or_buffer(
+                filepath_or_buffer, storage_options=storage_options
+            )
 
         if isinstance(filepath_or_buffer, self._workbook_class):
             self.book = filepath_or_buffer
@@ -837,14 +850,16 @@ class ExcelFile:
     from pandas.io.excel._pyxlsb import _PyxlsbReader
     from pandas.io.excel._xlrd import _XlrdReader
 
-    _engines = {
+    _engines: Mapping[str, Any] = {
         "xlrd": _XlrdReader,
         "openpyxl": _OpenpyxlReader,
         "odf": _ODFReader,
         "pyxlsb": _PyxlsbReader,
     }
 
-    def __init__(self, path_or_buffer, engine=None):
+    def __init__(
+        self, path_or_buffer, engine=None, storage_options: StorageOptions = None
+    ):
         if engine is None:
             engine = "xlrd"
             if isinstance(path_or_buffer, (BufferedIOBase, RawIOBase)):
@@ -858,13 +873,14 @@ def __init__(self, path_or_buffer, engine=None):
             raise ValueError(f"Unknown engine: {engine}")
 
         self.engine = engine
+        self.storage_options = storage_options
 
         # Could be a str, ExcelFile, Book, etc.
         self.io = path_or_buffer
         # Always a string
         self._io = stringify_path(path_or_buffer)
 
-        self._reader = self._engines[engine](self._io)
+        self._reader = self._engines[engine](self._io, storage_options=storage_options)
 
     def __fspath__(self):
         return self._io

pandas/io/excel/_odfreader.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from pandas._typing import FilePathOrBuffer, Scalar
+from pandas._typing import FilePathOrBuffer, Scalar, StorageOptions
 from pandas.compat._optional import import_optional_dependency
 
 import pandas as pd
@@ -16,13 +16,19 @@ class _ODFReader(_BaseExcelReader):
 
     Parameters
     ----------
-    filepath_or_buffer: string, path to be parsed or
+    filepath_or_buffer : string, path to be parsed or
         an open readable stream.
+    storage_options : StorageOptions
+        passed to fsspec for appropriate URLs (see ``get_filepath_or_buffer``)
     """
 
-    def __init__(self, filepath_or_buffer: FilePathOrBuffer):
+    def __init__(
+        self,
+        filepath_or_buffer: FilePathOrBuffer,
+        storage_options: StorageOptions = None,
+    ):
         import_optional_dependency("odf")
-        super().__init__(filepath_or_buffer)
+        super().__init__(filepath_or_buffer, storage_options=storage_options)
 
     @property
     def _workbook_class(self):

pandas/io/excel/_openpyxl.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from pandas._typing import FilePathOrBuffer, Scalar
+from pandas._typing import FilePathOrBuffer, Scalar, StorageOptions
 from pandas.compat._optional import import_optional_dependency
 
 from pandas.io.excel._base import ExcelWriter, _BaseExcelReader
@@ -467,17 +467,23 @@ def write_cells(
 
 
 class _OpenpyxlReader(_BaseExcelReader):
-    def __init__(self, filepath_or_buffer: FilePathOrBuffer) -> None:
+    def __init__(
+        self,
+        filepath_or_buffer: FilePathOrBuffer,
+        storage_options: StorageOptions = None,
+    ) -> None:
         """
         Reader using openpyxl engine.
 
         Parameters
         ----------
         filepath_or_buffer : string, path object or Workbook
             Object to be parsed.
+        storage_options : StorageOptions
+            passed to fsspec for appropriate URLs (see ``get_filepath_or_buffer``)
         """
         import_optional_dependency("openpyxl")
-        super().__init__(filepath_or_buffer)
+        super().__init__(filepath_or_buffer, storage_options=storage_options)
 
     @property
     def _workbook_class(self):

pandas/io/excel/_pyxlsb.py

@@ -1,25 +1,31 @@
 from typing import List
 
-from pandas._typing import FilePathOrBuffer, Scalar
+from pandas._typing import FilePathOrBuffer, Scalar, StorageOptions
 from pandas.compat._optional import import_optional_dependency
 
 from pandas.io.excel._base import _BaseExcelReader
 
 
 class _PyxlsbReader(_BaseExcelReader):
-    def __init__(self, filepath_or_buffer: FilePathOrBuffer):
+    def __init__(
+        self,
+        filepath_or_buffer: FilePathOrBuffer,
+        storage_options: StorageOptions = None,
+    ):
         """
         Reader using pyxlsb engine.
 
         Parameters
         ----------
-        filepath_or_buffer: str, path object, or Workbook
+        filepath_or_buffer : str, path object, or Workbook
             Object to be parsed.
+        storage_options : StorageOptions
+            passed to fsspec for appropriate URLs (see ``get_filepath_or_buffer``)
         """
         import_optional_dependency("pyxlsb")
         # This will call load_workbook on the filepath or buffer
         # And set the result to the book-attribute
-        super().__init__(filepath_or_buffer)
+        super().__init__(filepath_or_buffer, storage_options=storage_options)
 
     @property
     def _workbook_class(self):

pandas/io/excel/_xlrd.py

@@ -2,24 +2,27 @@
 
 import numpy as np
 
+from pandas._typing import StorageOptions
 from pandas.compat._optional import import_optional_dependency
 
 from pandas.io.excel._base import _BaseExcelReader
 
 
 class _XlrdReader(_BaseExcelReader):
-    def __init__(self, filepath_or_buffer):
+    def __init__(self, filepath_or_buffer, storage_options: StorageOptions = None):
         """
         Reader using xlrd engine.
 
         Parameters
         ----------
         filepath_or_buffer : string, path object or Workbook
             Object to be parsed.
+        storage_options : StorageOptions
+            passed to fsspec for appropriate URLs (see ``get_filepath_or_buffer``)
         """
         err_msg = "Install xlrd >= 1.0.0 for Excel support"
         import_optional_dependency("xlrd", extra=err_msg)
-        super().__init__(filepath_or_buffer)
+        super().__init__(filepath_or_buffer, storage_options=storage_options)
 
     @property
     def _workbook_class(self):

pandas/io/feather_format.py

@@ -1,13 +1,14 @@
 """ feather-format compat """
 
+from pandas._typing import StorageOptions
 from pandas.compat._optional import import_optional_dependency
 
 from pandas import DataFrame, Int64Index, RangeIndex
 
 from pandas.io.common import get_filepath_or_buffer
 
 
-def to_feather(df: DataFrame, path, storage_options=None, **kwargs):
+def to_feather(df: DataFrame, path, storage_options: StorageOptions = None, **kwargs):
     """
     Write a DataFrame to the binary Feather format.
 
@@ -77,7 +78,9 @@ def to_feather(df: DataFrame, path, storage_options=None, **kwargs):
     feather.write_feather(df, path, **kwargs)
 
 
-def read_feather(path, columns=None, use_threads: bool = True, storage_options=None):
+def read_feather(
+    path, columns=None, use_threads: bool = True, storage_options: StorageOptions = None
+):
     """
     Load a feather-format object from the file path.