From 5c5c5faef64017afe6235fd68df529ec5fd7dd9f Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 6 Dec 2018 00:31:29 +0100 Subject: [PATCH 1/7] CI/DEPS: fix boto errors and ResourceWarning; raise botocore min-ver --- ci/deps/travis-36.yaml | 1 + ci/deps/travis-37.yaml | 4 ++++ pandas/tests/io/conftest.py | 12 +++++++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ci/deps/travis-36.yaml b/ci/deps/travis-36.yaml index bfd69652730ed..f05a63bc2f157 100644 --- a/ci/deps/travis-36.yaml +++ b/ci/deps/travis-36.yaml @@ -4,6 +4,7 @@ channels: - conda-forge dependencies: - beautifulsoup4 + - botocore>=1.11 - cython>=0.28.2 - dask - fastparquet diff --git a/ci/deps/travis-37.yaml b/ci/deps/travis-37.yaml index a297786f6b14d..c503124d8cd26 100644 --- a/ci/deps/travis-37.yaml +++ b/ci/deps/travis-37.yaml @@ -5,6 +5,7 @@ channels: - c3i_test dependencies: - python=3.7 + - botocore>=1.11 - cython>=0.28.2 - numpy - python-dateutil @@ -14,3 +15,6 @@ dependencies: - pytest - pytest-xdist - hypothesis>=3.58.0 + - s3fs + - pip: + - moto diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 928519d39aed3..21184a16cc0c4 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,4 +1,7 @@ +import os + import pytest + from pandas.io.parsers import read_csv @@ -37,6 +40,12 @@ def s3_resource(tips_file, jsonl_file): """ pytest.importorskip('s3fs') boto3 = pytest.importorskip('boto3') + + # temporary workaround as moto fails for botocore >= 1.11 otherwise + # see https://github.com/spulec/moto/issues/1924 & 1952 + os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") + os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") + # GH-24092. See if boto.plugin skips the test or fails. try: pytest.importorskip("boto.plugin") @@ -59,7 +68,6 @@ def add_tips_files(bucket_name): Body=f) try: - s3 = moto.mock_s3() s3.start() @@ -73,7 +81,5 @@ def add_tips_files(bucket_name): conn.create_bucket(Bucket='cant_get_it', ACL='private') add_tips_files('cant_get_it') yield conn - except: # noqa: flake8 - pytest.skip("failure to use s3 resource") finally: s3.stop() From 7682581dfd2fb2510c78d1906f0eb3a538d7ee09 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 7 Dec 2018 17:37:38 +0100 Subject: [PATCH 2/7] Skip old botocore; don't skip on moto; undo env vars in finally --- pandas/tests/io/conftest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 21184a16cc0c4..fca37b75912cb 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,3 +1,4 @@ +from distutils.version import LooseVersion import os import pytest @@ -40,17 +41,18 @@ def s3_resource(tips_file, jsonl_file): """ pytest.importorskip('s3fs') boto3 = pytest.importorskip('boto3') + botocore = pytest.importorskip('botocore') + + if LooseVersion(botocore.__version__) < LooseVersion("1.11.0"): + # botocore leaks an uncatchable ResourceWarning before 1.11.0; + # see GH 23731 and https://github.com/boto/botocore/issues/1464 + pytest.skip("botocore is leaking resources before 1.11.0") # temporary workaround as moto fails for botocore >= 1.11 otherwise # see https://github.com/spulec/moto/issues/1924 & 1952 os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - # GH-24092. See if boto.plugin skips the test or fails. - try: - pytest.importorskip("boto.plugin") - except AttributeError: - raise pytest.skip("moto/moto error") moto = pytest.importorskip('moto') test_s3_files = [ @@ -83,3 +85,5 @@ def add_tips_files(bucket_name): yield conn finally: s3.stop() + os.environ.setdefault("AWS_ACCESS_KEY_ID", None) + os.environ.setdefault("AWS_SECRET_ACCESS_KEY", None) From 59793898f757f9da2ab5ee4e8c40ebb828e62fa3 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 7 Dec 2018 18:08:28 +0100 Subject: [PATCH 3/7] Try fix for moto import --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 03026647d6bb8..e0ab770ac46ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,6 +90,12 @@ before_install: - uname -a - git --version - git tag + # Because travis runs on Google Cloud and has a /etc/boto.cfg, + # it breaks moto import, see: + # https://github.com/spulec/moto/issues/1771 + # https://github.com/boto/boto/issues/3741 + # This overrides travis and tells it to look nowhere. + - export BOTO_CONFIG=/dev/null install: - echo "install start" From 49ff4540e629ee59fb8b6c6d7757766734e2c226 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 9 Dec 2018 15:12:20 +0100 Subject: [PATCH 4/7] Install moto through pip --- ci/deps/travis-36.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/travis-36.yaml b/ci/deps/travis-36.yaml index f05a63bc2f157..37cea281d48a6 100644 --- a/ci/deps/travis-36.yaml +++ b/ci/deps/travis-36.yaml @@ -36,10 +36,10 @@ dependencies: - pytest - pytest-xdist - pytest-cov - - moto - hypothesis>=3.58.0 - pip: - brotlipy - coverage + - moto - pandas-datareader - python-dateutil From eb1f65b1f4020cfddfd02d02c944a72f1a18c02e Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 10 Dec 2018 18:18:50 +0100 Subject: [PATCH 5/7] Add context manager for environment variables --- pandas/tests/io/conftest.py | 79 +++++++++++++++++++------------------ pandas/util/testing.py | 16 ++++++++ 2 files changed, 56 insertions(+), 39 deletions(-) diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index fca37b75912cb..af6f7ac4ef528 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -3,6 +3,8 @@ import pytest +import pandas.util.testing as tm + from pandas.io.parsers import read_csv @@ -48,42 +50,41 @@ def s3_resource(tips_file, jsonl_file): # see GH 23731 and https://github.com/boto/botocore/issues/1464 pytest.skip("botocore is leaking resources before 1.11.0") - # temporary workaround as moto fails for botocore >= 1.11 otherwise - # see https://github.com/spulec/moto/issues/1924 & 1952 - os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - - moto = pytest.importorskip('moto') - - test_s3_files = [ - ('tips.csv', tips_file), - ('tips.csv.gz', tips_file + '.gz'), - ('tips.csv.bz2', tips_file + '.bz2'), - ('items.jsonl', jsonl_file), - ] - - def add_tips_files(bucket_name): - for s3_key, file_name in test_s3_files: - with open(file_name, 'rb') as f: - conn.Bucket(bucket_name).put_object( - Key=s3_key, - Body=f) - - try: - s3 = moto.mock_s3() - s3.start() - - # see gh-16135 - bucket = 'pandas-test' - conn = boto3.resource("s3", region_name="us-east-1") - - conn.create_bucket(Bucket=bucket) - add_tips_files(bucket) - - conn.create_bucket(Bucket='cant_get_it', ACL='private') - add_tips_files('cant_get_it') - yield conn - finally: - s3.stop() - os.environ.setdefault("AWS_ACCESS_KEY_ID", None) - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", None) + with tm.ensure_safe_environment_variables(): + # temporary workaround as moto fails for botocore >= 1.11 otherwise, + # see https://github.com/spulec/moto/issues/1924 & 1952 + os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") + os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") + + moto = pytest.importorskip('moto') + + test_s3_files = [ + ('tips.csv', tips_file), + ('tips.csv.gz', tips_file + '.gz'), + ('tips.csv.bz2', tips_file + '.bz2'), + ('items.jsonl', jsonl_file), + ] + + def add_tips_files(bucket_name): + for s3_key, file_name in test_s3_files: + with open(file_name, 'rb') as f: + conn.Bucket(bucket_name).put_object( + Key=s3_key, + Body=f) + + try: + s3 = moto.mock_s3() + s3.start() + + # see gh-16135 + bucket = 'pandas-test' + conn = boto3.resource("s3", region_name="us-east-1") + + conn.create_bucket(Bucket=bucket) + add_tips_files(bucket) + + conn.create_bucket(Bucket='cant_get_it', ACL='private') + add_tips_files('cant_get_it') + yield conn + finally: + s3.stop() diff --git a/pandas/util/testing.py b/pandas/util/testing.py index faed4ccebd96b..8f02effd031c1 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -782,6 +782,22 @@ def ensure_clean_dir(): pass +@contextmanager +def ensure_safe_environment_variables(): + """ + Get a context manager to safely set environment variables + + All changes will be undone on close, hence environment variables set + within this contextmanager will neither persist nor change global state. + """ + saved_environ = dict(os.environ) + try: + yield + finally: + os.environ.clear() + os.environ.update(saved_environ) + + # ----------------------------------------------------------------------------- # Comparators From f660e4038ae3c34fc0bf2fcbeeda7cd931375df4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 14 Dec 2018 23:50:45 +0100 Subject: [PATCH 6/7] Add dependencies in environment.yml --- environment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/environment.yml b/environment.yml index e31511e5b8afe..370001dffe683 100644 --- a/environment.yml +++ b/environment.yml @@ -26,6 +26,8 @@ dependencies: # optional - beautifulsoup4>=4.2.1 - blosc + - botocore>=1.11 + - boto3 - bottleneck>=1.2.0 - fastparquet>=0.1.2 - html5lib @@ -41,6 +43,7 @@ dependencies: - pytables>=3.4.2 - pytest-cov - pytest-xdist + - s3fs - scipy>=1.1 - seaborn - sqlalchemy From b532696805fa2e36377738b64e5e8e46397bdc2e Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sat, 15 Dec 2018 00:36:34 +0100 Subject: [PATCH 7/7] also change requirements-dev.txt --- requirements-dev.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index facadf384f770..ae32c82aacd58 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,6 +15,8 @@ sphinx sphinxcontrib-spelling beautifulsoup4>=4.2.1 blosc +botocore>=1.11 +boto3 bottleneck>=1.2.0 fastparquet>=0.1.2 html5lib @@ -30,6 +32,7 @@ pyarrow>=0.7.0 tables>=3.4.2 pytest-cov pytest-xdist +s3fs scipy>=1.1 seaborn sqlalchemy