From 5f33d34dc114350ba0ee62028257f5e8f542ae34 Mon Sep 17 00:00:00 2001 From: Purna Chandra Mansingh Date: Fri, 11 Feb 2022 02:43:44 +0530 Subject: [PATCH 1/7] added storage_options description --- pandas/core/shared_docs.py | 5 ++++- pandas/io/common.py | 4 +++- pandas/io/excel/_base.py | 8 ++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pandas/core/shared_docs.py b/pandas/core/shared_docs.py index 35ee1c7a4ddbb..e3c0263206e84 100644 --- a/pandas/core/shared_docs.py +++ b/pandas/core/shared_docs.py @@ -402,7 +402,10 @@ host, port, username, password, etc. For HTTP(S) URLs the key-value pairs are forwarded to ``urllib`` as header options. For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to - ``fsspec``. Please see ``fsspec`` and ``urllib`` for more details.""" + ``fsspec``. Please see ``fsspec`` and ``urllib`` for more details, + and for more examples on storage options refer `here + `_.""" _shared_docs[ "compression_options" diff --git a/pandas/io/common.py b/pandas/io/common.py index 0331d320725ac..8db26866dd500 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -280,7 +280,9 @@ def _get_filepath_or_buffer( be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error will be raised if providing this argument with a local path or a file-like buffer. See the fsspec and backend storage implementation - docs for the set of allowed keys and values + docs for the set of allowed keys and values. For more examples on storage + options refer `here `_. .. versionadded:: 1.2.0 diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index c32f84d41ebde..044dfb0d5db22 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -278,7 +278,9 @@ be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error will be raised if providing this argument with a local path or a file-like buffer. See the fsspec and backend storage implementation - docs for the set of allowed keys and values. + docs for the set of allowed keys and values. For more examples on storage + options refer `here `_. .. versionadded:: 1.2.0 @@ -799,7 +801,9 @@ class ExcelWriter(metaclass=abc.ABCMeta): storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc., if using a URL that will - be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". + be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". For more + examples on storage options refer `here `_. .. versionadded:: 1.2.0 From 9905d5253494cba236a956a4bf005fed28dd1b92 Mon Sep 17 00:00:00 2001 From: Purna Chandra Mansingh Date: Fri, 11 Feb 2022 11:20:50 +0530 Subject: [PATCH 2/7] shared the doc-strings between all the methods --- pandas/io/common.py | 15 +++++---------- pandas/io/excel/_base.py | 8 ++------ pandas/io/excel/_odfreader.py | 6 ++++-- pandas/io/excel/_openpyxl.py | 7 +++++-- pandas/io/excel/_pyxlsb.py | 7 +++++-- pandas/io/excel/_xlrd.py | 7 +++++-- pandas/io/formats/xml.py | 9 +++++---- pandas/io/xml.py | 9 +++++---- 8 files changed, 36 insertions(+), 32 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index 8db26866dd500..d556952bfcd78 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -251,7 +251,10 @@ def is_fsspec_url(url: FilePath | BaseBuffer) -> bool: ) -@doc(compression_options=_shared_docs["compression_options"] % "filepath_or_buffer") +@doc( + storage_options=_shared_docs["storage_options"], + compression_options=_shared_docs["compression_options"] % "filepath_or_buffer", +) def _get_filepath_or_buffer( filepath_or_buffer: FilePath | BaseBuffer, encoding: str = "utf-8", @@ -274,15 +277,7 @@ def _get_filepath_or_buffer( encoding : the encoding to use to decode bytes, default is 'utf-8' mode : str, optional - storage_options : dict, optional - Extra options that make sense for a particular storage connection, e.g. - host, port, username, password, etc., if using a URL that will - be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error - will be raised if providing this argument with a local path or - a file-like buffer. See the fsspec and backend storage implementation - docs for the set of allowed keys and values. For more examples on storage - options refer `here `_. + {storage_options} .. versionadded:: 1.2.0 diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index 044dfb0d5db22..c32f84d41ebde 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -278,9 +278,7 @@ be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error will be raised if providing this argument with a local path or a file-like buffer. See the fsspec and backend storage implementation - docs for the set of allowed keys and values. For more examples on storage - options refer `here `_. + docs for the set of allowed keys and values. .. versionadded:: 1.2.0 @@ -801,9 +799,7 @@ class ExcelWriter(metaclass=abc.ABCMeta): storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc., if using a URL that will - be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". For more - examples on storage options refer `here `_. + be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". .. versionadded:: 1.2.0 diff --git a/pandas/io/excel/_odfreader.py b/pandas/io/excel/_odfreader.py index 952ad72b480b7..6adce02dc50f0 100644 --- a/pandas/io/excel/_odfreader.py +++ b/pandas/io/excel/_odfreader.py @@ -9,12 +9,15 @@ StorageOptions, ) from pandas.compat._optional import import_optional_dependency +from pandas.util._decorators import doc import pandas as pd +from pandas.core.shared_docs import _shared_docs from pandas.io.excel._base import BaseExcelReader +@doc(storage_options=_shared_docs["storage_options"]) class ODFReader(BaseExcelReader): """ Read tables out of OpenDocument formatted files. @@ -23,8 +26,7 @@ class ODFReader(BaseExcelReader): ---------- filepath_or_buffer : str, path to be parsed or an open readable stream. - storage_options : dict, optional - passed to fsspec for appropriate URLs (see ``_get_filepath_or_buffer``) + {storage_options} """ def __init__( diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 1ef4c348773bc..a2b1fbcce2b6d 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -18,6 +18,9 @@ WriteExcelBuffer, ) from pandas.compat._optional import import_optional_dependency +from pandas.util._decorators import doc + +from pandas.core.shared_docs import _shared_docs from pandas.io.excel._base import ( BaseExcelReader, @@ -526,6 +529,7 @@ def _write_cells( class OpenpyxlReader(BaseExcelReader): + @doc(storage_options=_shared_docs["storage_options"]) def __init__( self, filepath_or_buffer: FilePath | ReadBuffer[bytes], @@ -538,8 +542,7 @@ def __init__( ---------- filepath_or_buffer : str, path object or Workbook Object to be parsed. - storage_options : dict, optional - passed to fsspec for appropriate URLs (see ``_get_filepath_or_buffer``) + {storage_options} """ import_optional_dependency("openpyxl") super().__init__(filepath_or_buffer, storage_options=storage_options) diff --git a/pandas/io/excel/_pyxlsb.py b/pandas/io/excel/_pyxlsb.py index 9284cf917a48c..226361605df01 100644 --- a/pandas/io/excel/_pyxlsb.py +++ b/pandas/io/excel/_pyxlsb.py @@ -8,11 +8,15 @@ StorageOptions, ) from pandas.compat._optional import import_optional_dependency +from pandas.util._decorators import doc + +from pandas.core.shared_docs import _shared_docs from pandas.io.excel._base import BaseExcelReader class PyxlsbReader(BaseExcelReader): + @doc(storage_options=_shared_docs["storage_options"]) def __init__( self, filepath_or_buffer: FilePath | ReadBuffer[bytes], @@ -25,8 +29,7 @@ def __init__( ---------- filepath_or_buffer : str, path object, or Workbook Object to be parsed. - storage_options : dict, optional - passed to fsspec for appropriate URLs (see ``_get_filepath_or_buffer``) + {storage_options} """ import_optional_dependency("pyxlsb") # This will call load_workbook on the filepath or buffer diff --git a/pandas/io/excel/_xlrd.py b/pandas/io/excel/_xlrd.py index eea0f1c03b998..897011252f532 100644 --- a/pandas/io/excel/_xlrd.py +++ b/pandas/io/excel/_xlrd.py @@ -4,11 +4,15 @@ from pandas._typing import StorageOptions from pandas.compat._optional import import_optional_dependency +from pandas.util._decorators import doc + +from pandas.core.shared_docs import _shared_docs from pandas.io.excel._base import BaseExcelReader class XlrdReader(BaseExcelReader): + @doc(storage_options=_shared_docs["storage_options"]) def __init__(self, filepath_or_buffer, storage_options: StorageOptions = None): """ Reader using xlrd engine. @@ -17,8 +21,7 @@ def __init__(self, filepath_or_buffer, storage_options: StorageOptions = None): ---------- filepath_or_buffer : str, path object or Workbook Object to be parsed. - storage_options : dict, optional - passed to fsspec for appropriate URLs (see ``_get_filepath_or_buffer``) + {storage_options} """ err_msg = "Install xlrd >= 1.0.0 for Excel support" import_optional_dependency("xlrd", extra=err_msg) diff --git a/pandas/io/formats/xml.py b/pandas/io/formats/xml.py index aa69792cb1db0..a5e53edcdb02b 100644 --- a/pandas/io/formats/xml.py +++ b/pandas/io/formats/xml.py @@ -30,7 +30,10 @@ ) -@doc(compression_options=_shared_docs["compression_options"] % "path_or_buffer") +@doc( + storage_options=_shared_docs["storage_options"], + compression_options=_shared_docs["compression_options"] % "path_or_buffer", +) class BaseXMLFormatter: """ Subclass for formatting data in XML. @@ -82,9 +85,7 @@ class BaseXMLFormatter: .. versionchanged:: 1.4.0 Zstandard support. - storage_options : dict, optional - Extra options that make sense for a particular storage connection, - e.g. host, port, username, password, etc., + {storage_options} See also -------- diff --git a/pandas/io/xml.py b/pandas/io/xml.py index 8e463c94340c8..01535d0c65581 100644 --- a/pandas/io/xml.py +++ b/pandas/io/xml.py @@ -42,7 +42,10 @@ from pandas.io.parsers import TextParser -@doc(decompression_options=_shared_docs["decompression_options"] % "path_or_buffer") +@doc( + storage_options=_shared_docs["storage_options"], + decompression_options=_shared_docs["decompression_options"] % "path_or_buffer", +) class _XMLFrameParser: """ Internal subclass to parse XML into DataFrames. @@ -98,9 +101,7 @@ class _XMLFrameParser: .. versionchanged:: 1.4.0 Zstandard support. - storage_options : dict, optional - Extra options that make sense for a particular storage connection, - e.g. host, port, username, password, etc., + {storage_options} See also -------- From bb9408b8166cb8cad1395e19cb678bedd9f962a3 Mon Sep 17 00:00:00 2001 From: Purna Chandra Mansingh Date: Fri, 11 Feb 2022 11:26:54 +0530 Subject: [PATCH 3/7] shared the doc-strings between all the methods --- pandas/io/excel/_base.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index c32f84d41ebde..cf71b9214a881 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -274,11 +274,13 @@ are duplicate names in the columns. storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. - host, port, username, password, etc., if using a URL that will - be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error - will be raised if providing this argument with a local path or - a file-like buffer. See the fsspec and backend storage implementation - docs for the set of allowed keys and values. + host, port, username, password, etc. For HTTP(S) URLs the key-value pairs + are forwarded to ``urllib`` as header options. For other URLs (e.g. + starting with "s3://", and "gcs://") the key-value pairs are forwarded to + ``fsspec``. Please see ``fsspec`` and ``urllib`` for more details, + and for more examples on storage options refer `here + `_. .. versionadded:: 1.2.0 @@ -761,6 +763,7 @@ def parse( return output[asheetname] +@doc(storage_options=_shared_docs["storage_options"]) class ExcelWriter(metaclass=abc.ABCMeta): """ Class for writing DataFrame objects into excel sheets. @@ -796,10 +799,7 @@ class ExcelWriter(metaclass=abc.ABCMeta): (e.g. 'YYYY-MM-DD HH:MM:SS'). mode : {'w', 'a'}, default 'w' File mode to use (write or append). Append does not work with fsspec URLs. - storage_options : dict, optional - Extra options that make sense for a particular storage connection, e.g. - host, port, username, password, etc., if using a URL that will - be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". + {storage_options} .. versionadded:: 1.2.0 From a29af99c3782ddcf178392d0a8713e634b393841 Mon Sep 17 00:00:00 2001 From: Purna Chandra Mansingh Date: Fri, 11 Feb 2022 12:25:20 +0530 Subject: [PATCH 4/7] shared the doc-strings between all the methods --- pandas/io/excel/_base.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index cf71b9214a881..4bcc6ad51e41d 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -763,7 +763,6 @@ def parse( return output[asheetname] -@doc(storage_options=_shared_docs["storage_options"]) class ExcelWriter(metaclass=abc.ABCMeta): """ Class for writing DataFrame objects into excel sheets. @@ -799,7 +798,15 @@ class ExcelWriter(metaclass=abc.ABCMeta): (e.g. 'YYYY-MM-DD HH:MM:SS'). mode : {'w', 'a'}, default 'w' File mode to use (write or append). Append does not work with fsspec URLs. - {storage_options} + storage_options : dict, optional + Extra options that make sense for a particular storage connection, e.g. + host, port, username, password, etc. For HTTP(S) URLs the key-value pairs + are forwarded to ``urllib`` as header options. For other URLs (e.g. + starting with "s3://", and "gcs://") the key-value pairs are forwarded to + ``fsspec``. Please see ``fsspec`` and ``urllib`` for more details, + and for more examples on storage options refer `here + `_. .. versionadded:: 1.2.0 From 130c4fea065cdc84a7577f4f27548db59846dd53 Mon Sep 17 00:00:00 2001 From: Purna Chandra Mansingh Date: Sun, 13 Feb 2022 01:16:25 +0530 Subject: [PATCH 5/7] replaced {storage_options} in _base file --- pandas/io/excel/_base.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index 4bcc6ad51e41d..d28e7a52cb26f 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -272,15 +272,7 @@ Duplicate columns will be specified as 'X', 'X.1', ...'X.N', rather than 'X'...'X'. Passing in False will cause data to be overwritten if there are duplicate names in the columns. -storage_options : dict, optional - Extra options that make sense for a particular storage connection, e.g. - host, port, username, password, etc. For HTTP(S) URLs the key-value pairs - are forwarded to ``urllib`` as header options. For other URLs (e.g. - starting with "s3://", and "gcs://") the key-value pairs are forwarded to - ``fsspec``. Please see ``fsspec`` and ``urllib`` for more details, - and for more examples on storage options refer `here - `_. +{storage_options} .. versionadded:: 1.2.0 @@ -424,6 +416,7 @@ def read_excel( @deprecate_nonkeyword_arguments(allowed_args=["io", "sheet_name"], version="2.0") @Appender(_read_excel_doc) +@doc(storage_options=_shared_docs["storage_options"]) def read_excel( io, sheet_name: str | int | list[IntStrT] | None = 0, @@ -798,15 +791,7 @@ class ExcelWriter(metaclass=abc.ABCMeta): (e.g. 'YYYY-MM-DD HH:MM:SS'). mode : {'w', 'a'}, default 'w' File mode to use (write or append). Append does not work with fsspec URLs. - storage_options : dict, optional - Extra options that make sense for a particular storage connection, e.g. - host, port, username, password, etc. For HTTP(S) URLs the key-value pairs - are forwarded to ``urllib`` as header options. For other URLs (e.g. - starting with "s3://", and "gcs://") the key-value pairs are forwarded to - ``fsspec``. Please see ``fsspec`` and ``urllib`` for more details, - and for more examples on storage options refer `here - `_. + {storage_options} .. versionadded:: 1.2.0 From fc2c148dad482d5089332fea1fdba9a394e1604a Mon Sep 17 00:00:00 2001 From: Purna Chandra Mansingh Date: Sun, 13 Feb 2022 02:10:13 +0530 Subject: [PATCH 6/7] replaced {storage_options} in _base file --- pandas/io/excel/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index d28e7a52cb26f..95fe463d69c3c 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -414,9 +414,9 @@ def read_excel( ... +@doc(storage_options=_shared_docs["storage_options"]) @deprecate_nonkeyword_arguments(allowed_args=["io", "sheet_name"], version="2.0") @Appender(_read_excel_doc) -@doc(storage_options=_shared_docs["storage_options"]) def read_excel( io, sheet_name: str | int | list[IntStrT] | None = 0, @@ -756,6 +756,7 @@ def parse( return output[asheetname] +@doc(storage_options=_shared_docs["storage_options"]) class ExcelWriter(metaclass=abc.ABCMeta): """ Class for writing DataFrame objects into excel sheets. From fefb72e315e627f0b633a07edf0297acb8ea1d72 Mon Sep 17 00:00:00 2001 From: Purna Chandra Mansingh Date: Tue, 15 Feb 2022 18:30:45 +0530 Subject: [PATCH 7/7] fixed unescaped braces --- pandas/core/shared_docs.py | 8 ++++---- pandas/io/excel/_base.py | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pandas/core/shared_docs.py b/pandas/core/shared_docs.py index e3c0263206e84..0d7b9466a37f0 100644 --- a/pandas/core/shared_docs.py +++ b/pandas/core/shared_docs.py @@ -400,10 +400,10 @@ ] = """storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value pairs - are forwarded to ``urllib`` as header options. For other URLs (e.g. - starting with "s3://", and "gcs://") the key-value pairs are forwarded to - ``fsspec``. Please see ``fsspec`` and ``urllib`` for more details, - and for more examples on storage options refer `here + are forwarded to ``urllib.request.Request`` as header options. For other + URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are + forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more + details, and for more examples on storage options refer `here `_.""" diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py index 95fe463d69c3c..46ca93af1adf4 100644 --- a/pandas/io/excel/_base.py +++ b/pandas/io/excel/_base.py @@ -138,7 +138,7 @@ Append ``.squeeze("columns")`` to the call to ``read_excel`` to squeeze the data. dtype : Type name or dict of column -> type, default None - Data type for data or columns. E.g. {'a': np.float64, 'b': np.int32} + Data type for data or columns. E.g. {{'a': np.float64, 'b': np.int32}} Use `object` to preserve data as stored in Excel and not interpret dtype. If converters are specified, they will be applied INSTEAD of dtype conversion. @@ -222,7 +222,7 @@ each as a separate date column. * list of lists. e.g. If [[1, 3]] -> combine columns 1 and 3 and parse as a single date column. - * dict, e.g. {'foo' : [1, 3]} -> parse columns 1, 3 as date and call + * dict, e.g. {{'foo' : [1, 3]}} -> parse columns 1, 3 as date and call result 'foo' If a column or index contains an unparsable date, the entire column or @@ -318,7 +318,7 @@ Column types are inferred but can be explicitly specified >>> pd.read_excel('tmp.xlsx', index_col=0, -... dtype={'Name': str, 'Value': float}) # doctest: +SKIP +... dtype={{'Name': str, 'Value': float}}) # doctest: +SKIP Name Value 0 string1 1.0 1 string2 2.0 @@ -790,13 +790,13 @@ class ExcelWriter(metaclass=abc.ABCMeta): datetime_format : str, default None Format string for datetime objects written into Excel files. (e.g. 'YYYY-MM-DD HH:MM:SS'). - mode : {'w', 'a'}, default 'w' + mode : {{'w', 'a'}}, default 'w' File mode to use (write or append). Append does not work with fsspec URLs. {storage_options} .. versionadded:: 1.2.0 - if_sheet_exists : {'error', 'new', 'replace', 'overlay'}, default 'error' + if_sheet_exists : {{'error', 'new', 'replace', 'overlay'}}, default 'error' How to behave when trying to write to a sheet that already exists (append mode only). @@ -917,7 +917,7 @@ class ExcelWriter(metaclass=abc.ABCMeta): >>> with pd.ExcelWriter( ... "path_to_file.xlsx", ... engine="xlsxwriter", - ... engine_kwargs={"options": {"nan_inf_to_errors": True}} + ... engine_kwargs={{"options": {{"nan_inf_to_errors": True}}}} ... ) as writer: ... df.to_excel(writer) # doctest: +SKIP @@ -928,7 +928,7 @@ class ExcelWriter(metaclass=abc.ABCMeta): ... "path_to_file.xlsx", ... engine="openpyxl", ... mode="a", - ... engine_kwargs={"keep_vba": True} + ... engine_kwargs={{"keep_vba": True}} ... ) as writer: ... df.to_excel(writer, sheet_name="Sheet2") # doctest: +SKIP """