Storage options

doc/source/whatsnew/v1.1.0.rst

@@ -265,6 +265,12 @@ SSH, FTP, dropbox and github. For docs and capabilities, see the `fsspec docs`_.
 The existing capability to interface with S3 and GCS will be unaffected by this
 change, as ``fsspec`` will still bring in the same packages as before.
 
+Many read/write functions have acquired the `storage_options` optional argument,
+to pass a dictionary of parameters to the storage backend. This allows, for
+example, for passing credentials to S3 and GCS storage. The details of what
+parameters can be passed to which backends can be found in the documentation
+of the individual storage backends (linked from the fsspec docs).
+
 .. _Azure Datalake and Blob: https://github.com/dask/adlfs
 
 .. _fsspec docs: https://filesystem-spec.readthedocs.io/en/latest/

pandas/conftest.py

@@ -1224,3 +1224,21 @@ def sort_by_key(request):
     Tests None (no key) and the identity key.
     """
     return request.param
+
+
+@pytest.fixture()
+def fsspectest():
+    pytest.importorskip("fsspec")
+    from fsspec.implementations.memory import MemoryFileSystem
+    from fsspec import register_implementation
+
+    class TestMemoryFS(MemoryFileSystem):
+        protocol = "testmem"
+        test = [None]
+
+        def __init__(self, **kwargs):
+            self.test[0] = kwargs.pop("test", None)
+            super().__init__(**kwargs)
+
+    register_implementation("testmem", TestMemoryFS, True)
+    return TestMemoryFS()

pandas/core/frame.py

@@ -2055,6 +2055,7 @@ def to_stata(
         version: Optional[int] = 114,
         convert_strl: Optional[Sequence[Label]] = None,
         compression: Union[str, Mapping[str, str], None] = "infer",
+        storage_options: Optional[Dict[str, Any]] = None,
     ) -> None:
         """
         Export DataFrame object to Stata dta format.
@@ -2131,6 +2132,16 @@ def to_stata(
 
             .. versionadded:: 1.1.0
 
+        storage_options : dict, optional
+            Extra options that make sense for a particular storage connection, e.g.
+            host, port, username, password, etc., if using a URL that will
+            be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
+            will be raised if providing this argument with a local path or
+            a file-like buffer. See the fsspec and backend storage implementation
+            docs for the set of allowed keys and values
+
+            .. versionadded:: 1.1.0
+
         Raises
         ------
         NotImplementedError
@@ -2187,6 +2198,7 @@ def to_stata(
             write_index=write_index,
             variable_labels=variable_labels,
             compression=compression,
+            storage_options=storage_options,
             **kwargs,
         )
         writer.write_file()
@@ -2239,9 +2251,10 @@ def to_feather(self, path, **kwargs) -> None:
     )
     def to_markdown(
         self,
-        buf: Optional[IO[str]] = None,
-        mode: Optional[str] = None,
+        buf: Optional[Union[IO[str], str]] = None,
+        mode: str = "wt",
         index: bool = True,
+        storage_options: Optional[Dict[str, Any]] = None,
         **kwargs,
     ) -> Optional[str]:
         if "showindex" in kwargs:
@@ -2259,9 +2272,14 @@ def to_markdown(
         result = tabulate.tabulate(self, **kwargs)
         if buf is None:
             return result
-        buf, _, _, _ = get_filepath_or_buffer(buf, mode=mode)
+        buf, _, _, should_close = get_filepath_or_buffer(
+            buf, mode=mode, storage_options=storage_options
+        )
         assert buf is not None  # Help mypy.
+        assert not isinstance(buf, str)
         buf.writelines(result)
+        if should_close:
+            buf.close()
         return None
 
     @deprecate_kwarg(old_arg_name="fname", new_arg_name="path")
@@ -2272,6 +2290,7 @@ def to_parquet(
         compression: Optional[str] = "snappy",
         index: Optional[bool] = None,
         partition_cols: Optional[List[str]] = None,
+        storage_options: Optional[Dict[str, Any]] = None,
         **kwargs,
     ) -> None:
         """
@@ -2320,6 +2339,16 @@ def to_parquet(
 
             .. versionadded:: 0.24.0
 
+        storage_options : dict, optional
+            Extra options that make sense for a particular storage connection, e.g.
+            host, port, username, password, etc., if using a URL that will
+            be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
+            will be raised if providing this argument with a local path or
+            a file-like buffer. See the fsspec and backend storage implementation
+            docs for the set of allowed keys and values
+
+            .. versionadded:: 1.1.0
+
         **kwargs
             Additional arguments passed to the parquet library. See
             :ref:`pandas io <io.parquet>` for more details.
@@ -2366,6 +2395,7 @@ def to_parquet(
             compression=compression,
             index=index,
             partition_cols=partition_cols,
+            storage_options=storage_options,
             **kwargs,
         )
 

pandas/core/generic.py

@@ -2042,6 +2042,7 @@ def to_json(
         compression: Optional[str] = "infer",
         index: bool_t = True,
         indent: Optional[int] = None,
+        storage_options: Optional[Dict[str, Any]] = None,
     ) -> Optional[str]:
         """
         Convert the object to a JSON string.
@@ -2125,6 +2126,16 @@ def to_json(
 
            .. versionadded:: 1.0.0
 
+        storage_options : dict, optional
+            Extra options that make sense for a particular storage connection, e.g.
+            host, port, username, password, etc., if using a URL that will
+            be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
+            will be raised if providing this argument with a local path or
+            a file-like buffer. See the fsspec and backend storage implementation
+            docs for the set of allowed keys and values
+
+            .. versionadded:: 1.1.0
+
         Returns
         -------
         None or str
@@ -2303,6 +2314,7 @@ def to_json(
             compression=compression,
             index=index,
             indent=indent,
+            storage_options=storage_options,
         )
 
     def to_hdf(
@@ -2617,6 +2629,7 @@ def to_pickle(
         path,
         compression: Optional[str] = "infer",
         protocol: int = pickle.HIGHEST_PROTOCOL,
+        storage_options: Optional[Dict[str, Any]] = None,
     ) -> None:
         """
         Pickle (serialize) object to file.
@@ -2637,6 +2650,16 @@ def to_pickle(
 
             .. [1] https://docs.python.org/3/library/pickle.html.
 
+        storage_options : dict, optional
+            Extra options that make sense for a particular storage connection, e.g.
+            host, port, username, password, etc., if using a URL that will
+            be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
+            will be raised if providing this argument with a local path or
+            a file-like buffer. See the fsspec and backend storage implementation
+            docs for the set of allowed keys and values
+
+            .. versionadded:: 1.1.0
+
         See Also
         --------
         read_pickle : Load pickled pandas object (or any object) from file.
@@ -2670,7 +2693,13 @@ def to_pickle(
         """
         from pandas.io.pickle import to_pickle
 
-        to_pickle(self, path, compression=compression, protocol=protocol)
+        to_pickle(
+            self,
+            path,
+            compression=compression,
+            protocol=protocol,
+            storage_options=storage_options,
+        )
 
     def to_clipboard(
         self, excel: bool_t = True, sep: Optional[str] = None, **kwargs
@@ -3010,6 +3039,7 @@ def to_csv(
         escapechar: Optional[str] = None,
         decimal: Optional[str] = ".",
         errors: str = "strict",
+        storage_options: Optional[Dict[str, Any]] = None,
     ) -> Optional[str]:
         r"""
         Write object to a comma-separated values (csv) file.
@@ -3109,6 +3139,14 @@ def to_csv(
             See the errors argument for :func:`open` for a full list
             of options.
 
+        storage_options : dict, optional
+            Extra options that make sense for a particular storage connection, e.g.
+            host, port, username, password, etc., if using a URL that will
+            be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
+            will be raised if providing this argument with a local path or
+            a file-like buffer. See the fsspec and backend storage implementation
+            docs for the set of allowed keys and values
+
             .. versionadded:: 1.1.0
 
         Returns
@@ -3163,6 +3201,7 @@ def to_csv(
             doublequote=doublequote,
             escapechar=escapechar,
             decimal=decimal,
+            storage_options=storage_options,
         )
         formatter.save()
 

pandas/core/series.py

@@ -1421,7 +1421,7 @@ def to_string(
     def to_markdown(
         self,
         buf: Optional[IO[str]] = None,
-        mode: Optional[str] = None,
+        mode: str = "wt",
         index: bool = True,
         **kwargs,
     ) -> Optional[str]:
@@ -1435,7 +1435,7 @@ def to_markdown(
         buf : str, Path or StringIO-like, optional, default None
             Buffer to write to. If None, the output is returned as a string.
         mode : str, optional
-            Mode in which file is opened.
+            Mode in which file is opened, "wt" by default.
         index : bool, optional, default True
             Add index (row) labels.
 

pandas/io/common.py

@@ -167,8 +167,16 @@ def get_filepath_or_buffer(
     compression : {{'gzip', 'bz2', 'zip', 'xz', None}}, optional
     encoding : the encoding to use to decode bytes, default is 'utf-8'
     mode : str, optional
-    storage_options: dict, optional
-        passed on to fsspec, if using it; this is not yet accessed by the public API
+
+    storage_options : dict, optional
+        Extra options that make sense for a particular storage connection, e.g.
+        host, port, username, password, etc., if using a URL that will
+        be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
+        will be raised if providing this argument with a local path or
+        a file-like buffer. See the fsspec and backend storage implementation
+        docs for the set of allowed keys and values
+
+        .. versionadded:: 1.1.0
 
     Returns
     -------
@@ -180,6 +188,10 @@ def get_filepath_or_buffer(
 
     if isinstance(filepath_or_buffer, str) and is_url(filepath_or_buffer):
         # TODO: fsspec can also handle HTTP via requests, but leaving this unchanged
+        if storage_options:
+            raise ValueError(
+                "storage_options passed with file object or non-fsspec file path"
+            )
         req = urlopen(filepath_or_buffer)
         content_encoding = req.headers.get("Content-Encoding", None)
         if content_encoding == "gzip":
@@ -234,6 +246,10 @@ def get_filepath_or_buffer(
             ).open()
 
         return file_obj, encoding, compression, True
+    elif storage_options:
+        raise ValueError(
+            "storage_options passed with file object or non-fsspec file path"
+        )
 
     if isinstance(filepath_or_buffer, (str, bytes, mmap.mmap)):
         return _expand_user(filepath_or_buffer), None, compression, False

pandas/io/feather_format.py

@@ -4,17 +4,27 @@
 
 from pandas import DataFrame, Int64Index, RangeIndex
 
-from pandas.io.common import get_filepath_or_buffer, stringify_path
+from pandas.io.common import get_filepath_or_buffer
 
 
-def to_feather(df: DataFrame, path, **kwargs):
+def to_feather(df: DataFrame, path, storage_options=None, **kwargs):
     """
     Write a DataFrame to the binary Feather format.
 
     Parameters
     ----------
     df : DataFrame
     path : string file path, or file-like object
+
+    storage_options : dict, optional
+        Extra options that make sense for a particular storage connection, e.g.
+        host, port, username, password, etc., if using a URL that will
+        be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error
+        will be raised if providing this argument with a local path or
+        a file-like buffer. See the fsspec and backend storage implementation
+        docs for the set of allowed keys and values
+
+        .. versionadded:: 1.1.0
     **kwargs :
         Additional keywords passed to `pyarrow.feather.write_feather`.
 
@@ -23,7 +33,9 @@ def to_feather(df: DataFrame, path, **kwargs):
     import_optional_dependency("pyarrow")
     from pyarrow import feather
 
-    path = stringify_path(path)
+    path, _, _, should_close = get_filepath_or_buffer(
+        path, mode="wb", storage_options=storage_options
+    )
 
     if not isinstance(df, DataFrame):
         raise ValueError("feather only support IO with DataFrames")
@@ -64,7 +76,7 @@ def to_feather(df: DataFrame, path, **kwargs):
     feather.write_feather(df, path, **kwargs)
 
 
-def read_feather(path, columns=None, use_threads: bool = True):
+def read_feather(path, columns=None, use_threads: bool = True, storage_options=None):
     """
     Load a feather-format object from the file path.
 
@@ -98,7 +110,9 @@ def read_feather(path, columns=None, use_threads: bool = True):
     import_optional_dependency("pyarrow")
     from pyarrow import feather
 
-    path, _, _, should_close = get_filepath_or_buffer(path)
+    path, _, _, should_close = get_filepath_or_buffer(
+        path, storage_options=storage_options
+    )
 
     df = feather.read_feather(path, columns=columns, use_threads=bool(use_threads))
 

pandas/io/formats/csvs.py

@@ -5,7 +5,7 @@
 import csv as csvlib
 from io import StringIO
 import os
-from typing import Hashable, List, Mapping, Optional, Sequence, Union
+from typing import Any, Dict, Hashable, List, Mapping, Optional, Sequence, Union
 import warnings
 from zipfile import ZipFile
 
@@ -54,6 +54,7 @@ def __init__(
         doublequote: bool = True,
         escapechar: Optional[str] = None,
         decimal=".",
+        storage_options: Optional[Dict[str, Any]] = None,
     ):
         self.obj = obj
 
@@ -64,7 +65,11 @@ def __init__(
         compression, self.compression_args = get_compression_method(compression)
 
         self.path_or_buf, _, _, self.should_close = get_filepath_or_buffer(
-            path_or_buf, encoding=encoding, compression=compression, mode=mode
+            path_or_buf,
+            encoding=encoding,
+            compression=compression,
+            mode=mode,
+            storage_options=storage_options,
         )
         self.sep = sep
         self.na_rep = na_rep