Merge pull request #10604 from stephen-hoover/more-permissive-s3-reads

ENH: More permissive S3 reading

Author: Tom Augspurger

doc/source/whatsnew/v0.17.0.txt

@@ -389,3 +389,5 @@ Bug Fixes
 - Bug in operator equal on Index not being consistent with Series (:issue:`9947`)
 - Reading "famafrench" data via ``DataReader`` results in HTTP 404 error because of the website url is changed (:issue:`10591`).
 - Bug in `read_msgpack` where DataFrame to decode has duplicate column names (:issue:`9618`)
+
+- Bug in ``io.common.get_filepath_or_buffer`` which caused reading of valid S3 files to fail if the bucket also contained keys for which the user does not have read permission (:issue:`10604`)

pandas/io/common.py

@@ -151,7 +151,7 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None):
         except boto.exception.NoAuthHandlerFound:
             conn = boto.connect_s3(anon=True)
 
-        b = conn.get_bucket(parsed_url.netloc)
+        b = conn.get_bucket(parsed_url.netloc, validate=False)
         k = boto.s3.key.Key(b)
         k.key = parsed_url.path
         filepath_or_buffer = BytesIO(k.get_contents_as_string(

pandas/io/tests/test_parsers.py

@@ -4132,16 +4132,24 @@ def test_parse_public_s3_bucket(self):
         nt.assert_false(df.empty)
         tm.assert_frame_equal(pd.read_csv(tm.get_data_path('tips.csv')), df)
 
+        # Read public file from bucket with not-public contents
+        df = pd.read_csv('s3://cant_get_it/tips.csv')
+        nt.assert_true(isinstance(df, pd.DataFrame))
+        nt.assert_false(df.empty)
+        tm.assert_frame_equal(pd.read_csv(tm.get_data_path('tips.csv')), df)
+
     @tm.network
     def test_s3_fails(self):
         import boto
         with tm.assertRaisesRegexp(boto.exception.S3ResponseError,
                                 'S3ResponseError: 404 Not Found'):
             pd.read_csv('s3://nyqpug/asdf.csv')
 
+        # Receive a permission error when trying to read a private bucket.
+        # It's irrelevant here that this isn't actually a table.
         with tm.assertRaisesRegexp(boto.exception.S3ResponseError,
-                                'S3ResponseError: 403 Forbidden'):
-            pd.read_csv('s3://cant_get_it/tips.csv')
+                                   'S3ResponseError: 403 Forbidden'):
+            pd.read_csv('s3://cant_get_it/')
 
 
 def assert_same_values_and_dtype(res, exp):