From c9eab65bf79a4d87bd702bcd4e4bdfeb911edf9a Mon Sep 17 00:00:00 2001 From: Alistair Date: Thu, 26 Aug 2021 23:30:19 +0100 Subject: [PATCH 1/4] read_table needs storage_options arg --- pandas/io/parsers/readers.py | 1 + pandas/tests/io/test_fsspec.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py index c639a4a9d494e..1fe8c95d29c37 100644 --- a/pandas/io/parsers/readers.py +++ b/pandas/io/parsers/readers.py @@ -659,6 +659,7 @@ def read_table( low_memory=_c_parser_defaults["low_memory"], memory_map=False, float_precision=None, + storage_options: StorageOptions = None, ): # locals() should never be modified kwds = locals().copy() diff --git a/pandas/tests/io/test_fsspec.py b/pandas/tests/io/test_fsspec.py index eccfab3a31241..d2674732634f3 100644 --- a/pandas/tests/io/test_fsspec.py +++ b/pandas/tests/io/test_fsspec.py @@ -13,6 +13,7 @@ read_parquet, read_pickle, read_stata, + read_table, ) import pandas._testing as tm from pandas.util import _test_decorators as td @@ -122,6 +123,16 @@ def test_csv_options(fsspectest): assert fsspectest.test[0] == "csv_read" +def test_read_table_options(fsspectest): + df = DataFrame({"a": [0]}) + df.to_csv( + "testmem://test/test.csv", storage_options={"test": "csv_write"}, index=False + ) + assert fsspectest.test[0] == "csv_write" + read_table("testmem://test/test.csv", storage_options={"test": "csv_read"}) + assert fsspectest.test[0] == "csv_read" + + @pytest.mark.parametrize("extension", ["xlsx", "xls"]) def test_excel_options(fsspectest, extension): if extension == "xls": From 61e2f61fbc8123802a4eef2c2707134cec2b5939 Mon Sep 17 00:00:00 2001 From: Alistair Date: Fri, 27 Aug 2021 10:50:53 +0100 Subject: [PATCH 2/4] Make arg order consistent with read_csv --- pandas/io/parsers/readers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py index 1fe8c95d29c37..f1461e3b604a5 100644 --- a/pandas/io/parsers/readers.py +++ b/pandas/io/parsers/readers.py @@ -646,6 +646,7 @@ def read_table( escapechar=None, comment=None, encoding=None, + encoding_errors: str | None = "strict", dialect=None, # Error Handling error_bad_lines=None, @@ -653,7 +654,6 @@ def read_table( # TODO (2.0): set on_bad_lines to "error". # See _refine_defaults_read comment for why we do this. on_bad_lines=None, - encoding_errors: str | None = "strict", # Internal delim_whitespace=False, low_memory=_c_parser_defaults["low_memory"], From 2e82afface6d8a16b56e64e7ba81d380b88a96f7 Mon Sep 17 00:00:00 2001 From: Alistair Date: Fri, 27 Aug 2021 10:51:14 +0100 Subject: [PATCH 3/4] Add WhatsNew --- pandas/tests/io/test_fsspec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/io/test_fsspec.py b/pandas/tests/io/test_fsspec.py index d2674732634f3..2e495b2bcec18 100644 --- a/pandas/tests/io/test_fsspec.py +++ b/pandas/tests/io/test_fsspec.py @@ -124,6 +124,7 @@ def test_csv_options(fsspectest): def test_read_table_options(fsspectest): + # GH #39167 df = DataFrame({"a": [0]}) df.to_csv( "testmem://test/test.csv", storage_options={"test": "csv_write"}, index=False From 0e726bd157e79e515c289f1245839ae79d545d7e Mon Sep 17 00:00:00 2001 From: Alistair Date: Fri, 27 Aug 2021 10:51:50 +0100 Subject: [PATCH 4/4] WhatsNew --- doc/source/whatsnew/v1.4.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst index 205a49e7786a7..663b0a30aa48f 100644 --- a/doc/source/whatsnew/v1.4.0.rst +++ b/doc/source/whatsnew/v1.4.0.rst @@ -92,7 +92,7 @@ Other enhancements - :meth:`Series.sample`, :meth:`DataFrame.sample`, and :meth:`.GroupBy.sample` now accept a ``np.random.Generator`` as input to ``random_state``. A generator will be more performant, especially with ``replace=False`` (:issue:`38100`) - :meth:`Series.ewm`, :meth:`DataFrame.ewm`, now support a ``method`` argument with a ``'table'`` option that performs the windowing operation over an entire :class:`DataFrame`. See :ref:`Window Overview ` for performance and functional benefits (:issue:`42273`) - :meth:`.GroupBy.cummin` and :meth:`.GroupBy.cummax` now support the argument ``skipna`` (:issue:`34047`) -- +- :meth:`read_table` now supports the argument ``storage_options`` (:issue:`39167`) .. ---------------------------------------------------------------------------