CI for boto: fix errors; add coverage; add skip for uncatchable ResourceWarning (pandas-dev#23731)

Author: h-vetinari

.travis.yml

@@ -90,6 +90,12 @@ before_install:
   - uname -a
   - git --version
   - git tag
+  # Because travis runs on Google Cloud and has a /etc/boto.cfg,
+  # it breaks moto import, see:
+  # https://github.com/spulec/moto/issues/1771
+  # https://github.com/boto/boto/issues/3741
+  # This overrides travis and tells it to look nowhere.
+  - export BOTO_CONFIG=/dev/null
 
 install:
   - echo "install start"

ci/deps/travis-36.yaml

@@ -4,6 +4,7 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4
+  - botocore>=1.11
   - cython>=0.28.2
   - dask
   - fastparquet
@@ -35,10 +36,10 @@ dependencies:
   - pytest
   - pytest-xdist
   - pytest-cov
-  - moto
   - hypothesis>=3.58.0
   - pip:
     - brotlipy
     - coverage
+    - moto
     - pandas-datareader
     - python-dateutil

ci/deps/travis-37.yaml

@@ -5,6 +5,7 @@ channels:
   - c3i_test
 dependencies:
   - python=3.7
+  - botocore>=1.11
   - cython>=0.28.2
   - numpy
   - python-dateutil
@@ -14,3 +15,6 @@ dependencies:
   - pytest
   - pytest-xdist
   - hypothesis>=3.58.0
+  - s3fs
+  - pip:
+    - moto

environment.yml

@@ -27,6 +27,8 @@ dependencies:
   # optional
   - beautifulsoup4>=4.2.1
   - blosc
+  - botocore>=1.11
+  - boto3
   - bottleneck>=1.2.0
   - fastparquet>=0.1.2
   - html5lib
@@ -42,6 +44,7 @@ dependencies:
   - pytables>=3.4.2
   - pytest-cov
   - pytest-xdist
+  - s3fs
   - scipy>=1.1
   - seaborn
   - sqlalchemy

pandas/tests/io/conftest.py

@@ -1,4 +1,10 @@
+from distutils.version import LooseVersion
+import os
+
 import pytest
+
+import pandas.util.testing as tm
+
 from pandas.io.parsers import read_csv
 
 
@@ -37,43 +43,48 @@ def s3_resource(tips_file, jsonl_file):
     """
     pytest.importorskip('s3fs')
     boto3 = pytest.importorskip('boto3')
-    # GH-24092. See if boto.plugin skips the test or fails.
-    try:
-        pytest.importorskip("boto.plugin")
-    except AttributeError:
-        raise pytest.skip("moto/moto error")
-    moto = pytest.importorskip('moto')
-
-    test_s3_files = [
-        ('tips.csv', tips_file),
-        ('tips.csv.gz', tips_file + '.gz'),
-        ('tips.csv.bz2', tips_file + '.bz2'),
-        ('items.jsonl', jsonl_file),
-    ]
-
-    def add_tips_files(bucket_name):
-        for s3_key, file_name in test_s3_files:
-            with open(file_name, 'rb') as f:
-                conn.Bucket(bucket_name).put_object(
-                    Key=s3_key,
-                    Body=f)
-
-    try:
-
-        s3 = moto.mock_s3()
-        s3.start()
-
-        # see gh-16135
-        bucket = 'pandas-test'
-        conn = boto3.resource("s3", region_name="us-east-1")
-
-        conn.create_bucket(Bucket=bucket)
-        add_tips_files(bucket)
-
-        conn.create_bucket(Bucket='cant_get_it', ACL='private')
-        add_tips_files('cant_get_it')
-        yield conn
-    except:  # noqa: flake8
-        pytest.skip("failure to use s3 resource")
-    finally:
-        s3.stop()
+    botocore = pytest.importorskip('botocore')
+
+    if LooseVersion(botocore.__version__) < LooseVersion("1.11.0"):
+        # botocore leaks an uncatchable ResourceWarning before 1.11.0;
+        # see GH 23731 and https://github.com/boto/botocore/issues/1464
+        pytest.skip("botocore is leaking resources before 1.11.0")
+
+    with tm.ensure_safe_environment_variables():
+        # temporary workaround as moto fails for botocore >= 1.11 otherwise,
+        # see https://github.com/spulec/moto/issues/1924 & 1952
+        os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
+        os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
+
+        moto = pytest.importorskip('moto')
+
+        test_s3_files = [
+            ('tips.csv', tips_file),
+            ('tips.csv.gz', tips_file + '.gz'),
+            ('tips.csv.bz2', tips_file + '.bz2'),
+            ('items.jsonl', jsonl_file),
+        ]
+
+        def add_tips_files(bucket_name):
+            for s3_key, file_name in test_s3_files:
+                with open(file_name, 'rb') as f:
+                    conn.Bucket(bucket_name).put_object(
+                        Key=s3_key,
+                        Body=f)
+
+        try:
+            s3 = moto.mock_s3()
+            s3.start()
+
+            # see gh-16135
+            bucket = 'pandas-test'
+            conn = boto3.resource("s3", region_name="us-east-1")
+
+            conn.create_bucket(Bucket=bucket)
+            add_tips_files(bucket)
+
+            conn.create_bucket(Bucket='cant_get_it', ACL='private')
+            add_tips_files('cant_get_it')
+            yield conn
+        finally:
+            s3.stop()

pandas/util/testing.py

@@ -782,6 +782,22 @@ def ensure_clean_dir():
             pass
 
 
+@contextmanager
+def ensure_safe_environment_variables():
+    """
+    Get a context manager to safely set environment variables
+
+    All changes will be undone on close, hence environment variables set
+    within this contextmanager will neither persist nor change global state.
+    """
+    saved_environ = dict(os.environ)
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(saved_environ)
+
+
 # -----------------------------------------------------------------------------
 # Comparators
 

requirements-dev.txt

@@ -16,6 +16,8 @@ sphinxcontrib-spelling
 numpydoc
 beautifulsoup4>=4.2.1
 blosc
+botocore>=1.11
+boto3
 bottleneck>=1.2.0
 fastparquet>=0.1.2
 html5lib
@@ -31,6 +33,7 @@ pyarrow>=0.7.0
 tables>=3.4.2
 pytest-cov
 pytest-xdist
+s3fs
 scipy>=1.1
 seaborn
 sqlalchemy