From b3d61f74769dd9b4e71b656d47a7770d9600f4dd Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Dec 2018 01:05:38 +0000 Subject: [PATCH 1/5] WIP: Making explicity when we run the db tests --- .travis.yml | 6 +-- ci/azure/posix.yml | 8 ++-- ci/azure/windows.yml | 2 +- ci/deps/azure-27-compat.yaml | 3 -- ci/deps/azure-36-locale_slow.yaml | 3 -- ci/deps/azure-37-locale.yaml | 3 -- pandas/conftest.py | 5 +++ pandas/tests/io/test_sql.py | 75 +++++++++---------------------- pandas/util/_tester.py | 2 +- 9 files changed, 34 insertions(+), 73 deletions(-) diff --git a/.travis.yml b/.travis.yml index c30a7f3f58f55..4b99a38e57110 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,11 +34,11 @@ matrix: include: - dist: trusty env: - - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network" + - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network and not db" - dist: trusty env: - - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" PATTERN="slow" + - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" PATTERN="slow and not db" addons: apt: packages: @@ -55,7 +55,7 @@ matrix: - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true - dist: trusty env: - - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" PATTERN="not slow and not network" TEST_ARGS="-W error" PANDAS_TESTING_MODE="deprecate" + - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" PATTERN="not slow and not network and not db" TEST_ARGS="-W error" PANDAS_TESTING_MODE="deprecate" addons: apt: packages: diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 374a82a5ed7d0..94f08201d66ad 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -12,24 +12,24 @@ jobs: py35_np_120: ENV_FILE: ci/deps/azure-macos-35.yaml CONDA_PY: "35" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" ${{ if eq(parameters.name, 'Linux') }}: py27_np_120: ENV_FILE: ci/deps/azure-27-compat.yaml CONDA_PY: "27" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" py37_locale: ENV_FILE: ci/deps/azure-37-locale.yaml CONDA_PY: "37" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" LOCALE_OVERRIDE: "zh_CN.UTF-8" py36_locale_slow: ENV_FILE: ci/deps/azure-36-locale_slow.yaml CONDA_PY: "36" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" LOCALE_OVERRIDE: "it_IT.UTF-8" steps: diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index cece002024936..f06b229bb2656 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -38,7 +38,7 @@ jobs: displayName: 'Build' - script: | call activate pandas-dev - pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %* + pytest -m "not slow and not network and not db" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %* displayName: 'Test' - task: PublishTestResults@2 inputs: diff --git a/ci/deps/azure-27-compat.yaml b/ci/deps/azure-27-compat.yaml index f3cc615c35243..8899e22bdf6cf 100644 --- a/ci/deps/azure-27-compat.yaml +++ b/ci/deps/azure-27-compat.yaml @@ -9,13 +9,11 @@ dependencies: - numexpr=2.6.1 - numpy=1.12.0 - openpyxl=2.5.5 - - psycopg2 - pytables=3.4.2 - python-dateutil=2.5.0 - python=2.7* - pytz=2013b - scipy=0.18.1 - - sqlalchemy=0.7.8 - xlrd=1.0.0 - xlsxwriter=0.5.2 - xlwt=0.7.5 @@ -25,5 +23,4 @@ dependencies: - pip: - html5lib==1.0b2 - beautifulsoup4==4.2.1 - - pymysql==0.6.0 - hypothesis>=3.58.0 diff --git a/ci/deps/azure-36-locale_slow.yaml b/ci/deps/azure-36-locale_slow.yaml index 4bbc6a2c11f1e..c7d2334623501 100644 --- a/ci/deps/azure-36-locale_slow.yaml +++ b/ci/deps/azure-36-locale_slow.yaml @@ -15,15 +15,12 @@ dependencies: - numexpr - numpy - openpyxl - - psycopg2 - - pymysql - pytables - python-dateutil - python=3.6* - pytz - s3fs - scipy - - sqlalchemy - xarray - xlrd - xlsxwriter diff --git a/ci/deps/azure-37-locale.yaml b/ci/deps/azure-37-locale.yaml index 11a698ce7648e..b5a05c49b8083 100644 --- a/ci/deps/azure-37-locale.yaml +++ b/ci/deps/azure-37-locale.yaml @@ -14,15 +14,12 @@ dependencies: - numexpr - numpy - openpyxl - - psycopg2 - - pymysql - pytables - python-dateutil - python=3.7* - pytz - s3fs - scipy - - sqlalchemy - xarray - xlrd - xlsxwriter diff --git a/pandas/conftest.py b/pandas/conftest.py index c72efdf865052..5aa644bb942f8 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -34,6 +34,8 @@ def pytest_addoption(parser): help="skip slow tests") parser.addoption("--skip-network", action="store_true", help="skip network tests") + parser.addoption("--skip-db", action="store_true", + help="skip db tests") parser.addoption("--run-high-memory", action="store_true", help="run high memory tests") parser.addoption("--only-slow", action="store_true", @@ -52,6 +54,9 @@ def pytest_runtest_setup(item): if 'network' in item.keywords and item.config.getoption("--skip-network"): pytest.skip("skipping due to --skip-network") + if 'db' in item.keywords and item.config.getoption("--skip-db"): + pytest.skip("skipping due to --skip-db") + if 'high_memory' in item.keywords and not item.config.getoption( "--run-high-memory"): pytest.skip( diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index eeeb55cb8e70c..7f141e8de101f 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1860,11 +1860,8 @@ def connect(cls): @classmethod def setup_driver(cls): - try: - import psycopg2 # noqa - cls.driver = 'psycopg2' - except ImportError: - pytest.skip('psycopg2 not installed') + import psycopg2 # noqa + cls.driver = 'psycopg2' def test_schema_support(self): # only test this for postgresql (schema's not supported in @@ -1932,21 +1929,25 @@ def test_schema_support(self): @pytest.mark.single +@pytest.mark.db class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy): pass @pytest.mark.single +@pytest.mark.db class TestMySQLAlchemyConn(_TestMySQLAlchemy, _TestSQLAlchemyConn): pass @pytest.mark.single +@pytest.mark.db class TestPostgreSQLAlchemy(_TestPostgreSQLAlchemy, _TestSQLAlchemy): pass @pytest.mark.single +@pytest.mark.db class TestPostgreSQLAlchemyConn(_TestPostgreSQLAlchemy, _TestSQLAlchemyConn): pass @@ -2189,13 +2190,6 @@ def tquery(query, con=None, cur=None): return list(res) -def _skip_if_no_pymysql(): - try: - import pymysql # noqa - except ImportError: - pytest.skip('pymysql not installed, skipping') - - @pytest.mark.single class TestXSQLite(SQLiteMixIn): @@ -2404,76 +2398,56 @@ def clean_up(test_table_to_drop): @pytest.mark.single +@pytest.mark.db @pytest.mark.skip(reason="gh-13611: there is no support for MySQL " "if SQLAlchemy is not installed") class TestXMySQL(MySQLMixIn): @pytest.fixture(autouse=True, scope='class') def setup_class(cls): - _skip_if_no_pymysql() - - # test connection import pymysql - try: - # Try Travis defaults. - # No real user should allow root access with a blank password. - pymysql.connect(host='localhost', user='root', passwd='', - db='pandas_nosetest') - except pymysql.Error: - pass - else: - return + pymysql.connect(host='localhost', user='root', passwd='', + db='pandas_nosetest') try: pymysql.connect(read_default_group='pandas') except pymysql.ProgrammingError: - pytest.skip( + raise RuntimeError( "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " - "typically located at ~/.my.cnf or /etc/.my.cnf. ") + "typically located at ~/.my.cnf or /etc/.my.cnf.") except pymysql.Error: - pytest.skip( + raise RuntimeError( "Cannot connect to database. " "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " - "typically located at ~/.my.cnf or /etc/.my.cnf. ") + "typically located at ~/.my.cnf or /etc/.my.cnf.") @pytest.fixture(autouse=True) def setup_method(self, request, datapath): - _skip_if_no_pymysql() import pymysql + pymysql.connect(host='localhost', user='root', passwd='', + db='pandas_nosetest') try: - # Try Travis defaults. - # No real user should allow root access with a blank password. - self.conn = pymysql.connect(host='localhost', user='root', - passwd='', db='pandas_nosetest') - except pymysql.Error: - pass - else: - return - try: - self.conn = pymysql.connect(read_default_group='pandas') + pymysql.connect(read_default_group='pandas') except pymysql.ProgrammingError: - pytest.skip( + raise RuntimeError( "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " - "typically located at ~/.my.cnf or /etc/.my.cnf. ") + "typically located at ~/.my.cnf or /etc/.my.cnf.") except pymysql.Error: - pytest.skip( + raise RuntimeError( "Cannot connect to database. " "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " - "typically located at ~/.my.cnf or /etc/.my.cnf. ") + "typically located at ~/.my.cnf or /etc/.my.cnf.") self.method = request.function def test_basic(self): - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() self._check_roundtrip(frame) def test_write_row_by_row(self): - - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() frame.iloc[0, 0] = np.nan drop_sql = "DROP TABLE IF EXISTS test" @@ -2493,7 +2467,6 @@ def test_write_row_by_row(self): tm.assert_frame_equal(result, frame, check_less_precise=True) def test_chunksize_read_type(self): - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() frame.index.name = "index" drop_sql = "DROP TABLE IF EXISTS test" @@ -2508,7 +2481,6 @@ def test_chunksize_read_type(self): tm.assert_frame_equal(frame[:chunksize], chunk_df) def test_execute(self): - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() drop_sql = "DROP TABLE IF EXISTS test" create_sql = sql.get_schema(frame, 'test') @@ -2528,7 +2500,6 @@ def test_execute(self): tm.assert_frame_equal(result, frame[:1]) def test_schema(self): - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() create_sql = sql.get_schema(frame, 'test') lines = create_sql.splitlines() @@ -2548,7 +2519,6 @@ def test_schema(self): @tm.capture_stdout def test_execute_fail(self): - _skip_if_no_pymysql() drop_sql = "DROP TABLE IF EXISTS test" create_sql = """ CREATE TABLE test @@ -2570,7 +2540,6 @@ def test_execute_fail(self): sql.execute('INSERT INTO test VALUES("foo", "bar", 7)', self.conn) def test_execute_closed_connection(self, request, datapath): - _skip_if_no_pymysql() drop_sql = "DROP TABLE IF EXISTS test" create_sql = """ CREATE TABLE test @@ -2595,11 +2564,9 @@ def test_execute_closed_connection(self, request, datapath): self.setup_method(request, datapath) def test_na_roundtrip(self): - _skip_if_no_pymysql() pass def _check_roundtrip(self, frame): - _skip_if_no_pymysql() drop_sql = "DROP TABLE IF EXISTS test_table" cur = self.conn.cursor() with warnings.catch_warnings(): @@ -2636,13 +2603,11 @@ def _check_roundtrip(self, frame): tm.assert_frame_equal(expected, result) def test_keyword_as_column_names(self): - _skip_if_no_pymysql() df = DataFrame({'From': np.ones(5)}) sql.to_sql(df, con=self.conn, name='testkeywords', if_exists='replace', index=False) def test_if_exists(self): - _skip_if_no_pymysql() df_if_exists_1 = DataFrame({'col1': [1, 2], 'col2': ['A', 'B']}) df_if_exists_2 = DataFrame( {'col1': [3, 4, 5], 'col2': ['C', 'D', 'E']}) diff --git a/pandas/util/_tester.py b/pandas/util/_tester.py index aad2f00fa0478..18e8d415459fd 100644 --- a/pandas/util/_tester.py +++ b/pandas/util/_tester.py @@ -16,7 +16,7 @@ def test(extra_args=None): import hypothesis # noqa except ImportError: raise ImportError("Need hypothesis>=3.58 to run tests") - cmd = ['--skip-slow', '--skip-network'] + cmd = ['--skip-slow', '--skip-network', '--skip-db'] if extra_args: if not isinstance(extra_args, list): extra_args = [extra_args] From 36406be57041dd58171577be45151b6b3a4d51f0 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Dec 2018 01:30:53 +0000 Subject: [PATCH 2/5] Registering db marker --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 380100df774c1..929d2ebbcb918 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,6 +68,7 @@ markers = single: mark a test as single cpu only slow: mark a test as slow network: mark a test as network + db: tests requiring a database (mysql or postgres) high_memory: mark a test as a high-memory only clipboard: mark a pd.read_clipboard test doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL From 5ff2307a1936c62ae2e0e03b5d1c2cb9497bcf8c Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Thu, 27 Dec 2018 14:27:35 +0000 Subject: [PATCH 3/5] Skipping db tests if pymysql or psycopg are not installed --- pandas/tests/io/test_sql.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 7f141e8de101f..0bf4cf8496edd 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1930,24 +1930,28 @@ def test_schema_support(self): @pytest.mark.single @pytest.mark.db +@pytest.importorskip('pymysql') class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy): pass @pytest.mark.single @pytest.mark.db +@pytest.importorskip('pymysql') class TestMySQLAlchemyConn(_TestMySQLAlchemy, _TestSQLAlchemyConn): pass @pytest.mark.single @pytest.mark.db +@pytest.importorskip('psycopg2') class TestPostgreSQLAlchemy(_TestPostgreSQLAlchemy, _TestSQLAlchemy): pass @pytest.mark.single @pytest.mark.db +@pytest.importorskip('psycopg2') class TestPostgreSQLAlchemyConn(_TestPostgreSQLAlchemy, _TestSQLAlchemyConn): pass @@ -2401,6 +2405,7 @@ def clean_up(test_table_to_drop): @pytest.mark.db @pytest.mark.skip(reason="gh-13611: there is no support for MySQL " "if SQLAlchemy is not installed") +@pytest.importorskip('pymysql') class TestXMySQL(MySQLMixIn): @pytest.fixture(autouse=True, scope='class') From 9fd127e6b687674a6846408405fbbafe653f799c Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 28 Dec 2018 00:52:01 +0000 Subject: [PATCH 4/5] Fixing importorskip calls, and making tests fail if connection to database fails --- pandas/tests/io/test_sql.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 0bf4cf8496edd..6d585628f3e71 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1152,14 +1152,8 @@ class _TestSQLAlchemy(SQLAlchemyMixIn, PandasSQLTest): def setup_class(cls): cls.setup_import() cls.setup_driver() - - # test connection - try: - conn = cls.connect() - conn.connect() - except sqlalchemy.exc.OperationalError: - msg = "{0} - can't connect to {1} server".format(cls, cls.flavor) - pytest.skip(msg) + conn = cls.connect() + conn.connect() def load_test_data_and_sql(self): self._load_raw_sql() @@ -1791,13 +1785,10 @@ def connect(cls): @classmethod def setup_driver(cls): - try: - import pymysql # noqa - cls.driver = 'pymysql' - from pymysql.constants import CLIENT - cls.connect_args = {'client_flag': CLIENT.MULTI_STATEMENTS} - except ImportError: - pytest.skip('pymysql not installed') + pymysql = pytest.importorskip('pymysql') + cls.driver = 'pymysql' + cls.connect_args = { + 'client_flag': pymysql.constants.CLIENT.MULTI_STATEMENTS} def test_default_type_conversion(self): df = sql.read_sql_table("types_test_data", self.conn) @@ -1860,7 +1851,7 @@ def connect(cls): @classmethod def setup_driver(cls): - import psycopg2 # noqa + psycopg2 = pytest.importorskip('psycopg2') cls.driver = 'psycopg2' def test_schema_support(self): @@ -1930,28 +1921,24 @@ def test_schema_support(self): @pytest.mark.single @pytest.mark.db -@pytest.importorskip('pymysql') class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy): pass @pytest.mark.single @pytest.mark.db -@pytest.importorskip('pymysql') class TestMySQLAlchemyConn(_TestMySQLAlchemy, _TestSQLAlchemyConn): pass @pytest.mark.single @pytest.mark.db -@pytest.importorskip('psycopg2') class TestPostgreSQLAlchemy(_TestPostgreSQLAlchemy, _TestSQLAlchemy): pass @pytest.mark.single @pytest.mark.db -@pytest.importorskip('psycopg2') class TestPostgreSQLAlchemyConn(_TestPostgreSQLAlchemy, _TestSQLAlchemyConn): pass @@ -2405,12 +2392,11 @@ def clean_up(test_table_to_drop): @pytest.mark.db @pytest.mark.skip(reason="gh-13611: there is no support for MySQL " "if SQLAlchemy is not installed") -@pytest.importorskip('pymysql') class TestXMySQL(MySQLMixIn): @pytest.fixture(autouse=True, scope='class') def setup_class(cls): - import pymysql + pymysql = pytest.importorskip('pymysql') pymysql.connect(host='localhost', user='root', passwd='', db='pandas_nosetest') try: @@ -2429,7 +2415,7 @@ def setup_class(cls): @pytest.fixture(autouse=True) def setup_method(self, request, datapath): - import pymysql + pymysql = pytest.importorskip('pymysql') pymysql.connect(host='localhost', user='root', passwd='', db='pandas_nosetest') try: From d9ac03e10767325778e2e166be60a0ddee0e5ba8 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 28 Dec 2018 01:52:57 +0000 Subject: [PATCH 5/5] Removing unused variable --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 6d585628f3e71..8f2a212b457a4 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1851,7 +1851,7 @@ def connect(cls): @classmethod def setup_driver(cls): - psycopg2 = pytest.importorskip('psycopg2') + pytest.importorskip('psycopg2') cls.driver = 'psycopg2' def test_schema_support(self):