CI: Bump s3fs (#29573)

Author: TomAugspurger

ci/deps/travis-36-cov.yaml

@@ -29,7 +29,7 @@ dependencies:
   - python-snappy
   - python=3.6.*
   - pytz
-  - s3fs<0.3
+  - s3fs
   - scikit-learn
   - scipy
   - sqlalchemy

ci/deps/travis-36-locale.yaml

@@ -26,7 +26,7 @@ dependencies:
   - python-dateutil
   - python=3.6.*
   - pytz
-  - s3fs=0.0.8
+  - s3fs=0.3.0
   - scipy
   - sqlalchemy=1.1.4
   - xarray=0.10

ci/deps/travis-36-slow.yaml

@@ -18,7 +18,7 @@ dependencies:
   - python-dateutil
   - python=3.6.*
   - pytz
-  - s3fs<0.3
+  - s3fs
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/travis-37.yaml

@@ -17,7 +17,7 @@ dependencies:
   - pytest-xdist>=1.29.0
   - pytest-mock
   - hypothesis>=3.58.0
-  - s3fs<0.3
+  - s3fs
   - pip
   - pyreadstat
   - pip:

doc/source/getting_started/install.rst

@@ -263,7 +263,7 @@ pymysql                   0.7.11             MySQL engine for sqlalchemy
 pyreadstat                                   SPSS files (.sav) reading
 pytables                  3.4.2              HDF5 reading / writing
 qtpy                                         Clipboard I/O
-s3fs                      0.0.8              Amazon S3 access
+s3fs                      0.3.0              Amazon S3 access
 xarray                    0.8.2              pandas-like API for N-dimensional data
 xclip                                        Clipboard I/O on linux
 xlrd                      1.1.0              Excel reading

doc/source/whatsnew/v1.0.0.rst

@@ -183,6 +183,7 @@ Backwards incompatible API changes
 Other API changes
 ^^^^^^^^^^^^^^^^^
 
+- Bumpded the minimum supported version of ``s3fs`` from 0.0.8 to 0.3.0 (:issue:`28616`)
 - :class:`pandas.core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`)
 - :meth:`pandas.api.types.infer_dtype` will now return "integer-na" for integer and ``np.nan`` mix (:issue:`27283`)
 - :meth:`MultiIndex.from_arrays` will no longer infer names from arrays if ``names=None`` is explicitly provided (:issue:`27292`)

pandas/compat/_optional.py

@@ -18,7 +18,7 @@
     "pandas_gbq": "0.8.0",
     "pyarrow": "0.9.0",
     "pytables": "3.4.2",
-    "s3fs": "0.0.8",
+    "s3fs": "0.3.0",
     "scipy": "0.19.0",
     "sqlalchemy": "1.1.4",
     "tables": "3.4.2",

pandas/tests/io/conftest.py

@@ -40,7 +40,7 @@ def s3_resource(tips_file, jsonl_file):
     A private bucket "cant_get_it" is also created. The boto3 s3 resource
     is yielded by the fixture.
     """
-    pytest.importorskip("s3fs")
+    s3fs = pytest.importorskip("s3fs")
     boto3 = pytest.importorskip("boto3")
 
     with tm.ensure_safe_environment_variables():
@@ -77,6 +77,7 @@ def add_tips_files(bucket_name):
 
             conn.create_bucket(Bucket="cant_get_it", ACL="private")
             add_tips_files("cant_get_it")
+            s3fs.S3FileSystem.clear_instance_cache()
             yield conn
         finally:
             s3.stop()

pandas/tests/io/parser/test_network.py

@@ -166,7 +166,7 @@ def test_s3_fails(self):
         # Receive a permission error when trying to read a private bucket.
         # It's irrelevant here that this isn't actually a table.
         with pytest.raises(IOError):
-            read_csv("s3://cant_get_it/")
+            read_csv("s3://cant_get_it/file.csv")
 
     def test_read_csv_handles_boto_s3_object(self, s3_resource, tips_file):
         # see gh-16135
@@ -184,6 +184,8 @@ def test_read_csv_handles_boto_s3_object(self, s3_resource, tips_file):
 
     def test_read_csv_chunked_download(self, s3_resource, caplog):
         # 8 MB, S3FS usees 5MB chunks
+        import s3fs
+
         df = DataFrame(np.random.randn(100000, 4), columns=list("abcd"))
         buf = BytesIO()
         str_buf = StringIO()
@@ -194,7 +196,13 @@ def test_read_csv_chunked_download(self, s3_resource, caplog):
 
         s3_resource.Bucket("pandas-test").put_object(Key="large-file.csv", Body=buf)
 
-        with caplog.at_level(logging.DEBUG, logger="s3fs.core"):
+        # Possibly some state leaking in between tests.
+        # If we don't clear this cache, we saw `GetObject operation: Forbidden`.
+        # Presumably the s3fs instance is being cached, with the directory listing
+        # from *before* we add the large-file.csv in the pandas-test bucket.
+        s3fs.S3FileSystem.clear_instance_cache()
+
+        with caplog.at_level(logging.DEBUG, logger="s3fs"):
             read_csv("s3://pandas-test/large-file.csv", nrows=5)
             # log of fetch_range (start, stop)
             assert (0, 5505024) in {x.args[-2:] for x in caplog.records}