TST: Making explicit when we run the db tests (pandas-dev#24450)

Author: datapythonista

.travis.yml

@@ -34,7 +34,7 @@ matrix:
     include:
     - dist: trusty
       env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network and not db"
 
     - dist: trusty
       env:

ci/azure/posix.yml

@@ -12,37 +12,37 @@ jobs:
         py35_np_120:
           ENV_FILE: ci/deps/azure-macos-35.yaml
           CONDA_PY: "35"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not db"
 
       ${{ if eq(parameters.name, 'Linux') }}:
         py27_np_120:
           ENV_FILE: ci/deps/azure-27-compat.yaml
           CONDA_PY: "27"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not db"
 
         py27_locale_slow_old_np:
           ENV_FILE: ci/deps/azure-27-locale.yaml
           CONDA_PY: "27"
-          PATTERN: "slow"
+          PATTERN: "slow and not db"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
           EXTRA_APT: "language-pack-zh-hans"
 
         py36_locale_slow:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not db"
           LOCALE_OVERRIDE: "it_IT.UTF-8"
 
         py37_locale:
           ENV_FILE: ci/deps/azure-37-locale.yaml
           CONDA_PY: "37"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not db"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
         py37_np_dev:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml
           CONDA_PY: "37"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not db"
           TEST_ARGS: "-W error"
           PANDAS_TESTING_MODE: "deprecate"
           EXTRA_APT: "xsel"

ci/azure/windows.yml

@@ -38,7 +38,7 @@ jobs:
       displayName: 'Build'
     - script: |
         call activate pandas-dev
-        pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
+        pytest -m "not slow and not network and not db" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
       displayName: 'Test'
     - task: PublishTestResults@2
       inputs:

ci/deps/azure-27-compat.yaml

@@ -9,13 +9,11 @@ dependencies:
   - numexpr=2.6.1
   - numpy=1.12.0
   - openpyxl=2.5.5
-  - psycopg2
   - pytables=3.4.2
   - python-dateutil=2.5.0
   - python=2.7*
   - pytz=2013b
   - scipy=0.18.1
-  - sqlalchemy=0.7.8
   - xlrd=1.0.0
   - xlsxwriter=0.5.2
   - xlwt=0.7.5
@@ -25,5 +23,4 @@ dependencies:
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1
-    - pymysql==0.6.0
     - hypothesis>=3.58.0

ci/deps/azure-36-locale_slow.yaml

@@ -15,15 +15,12 @@ dependencies:
   - numexpr
   - numpy
   - openpyxl
-  - psycopg2
-  - pymysql
   - pytables
   - python-dateutil
   - python=3.6*
   - pytz
   - s3fs
   - scipy
-  - sqlalchemy
   - xarray
   - xlrd
   - xlsxwriter

ci/deps/azure-37-locale.yaml

@@ -14,15 +14,12 @@ dependencies:
   - numexpr
   - numpy
   - openpyxl
-  - psycopg2
-  - pymysql
   - pytables
   - python-dateutil
   - python=3.7*
   - pytz
   - s3fs
   - scipy
-  - sqlalchemy
   - xarray
   - xlrd
   - xlsxwriter

pandas/conftest.py

@@ -34,6 +34,8 @@ def pytest_addoption(parser):
                      help="skip slow tests")
     parser.addoption("--skip-network", action="store_true",
                      help="skip network tests")
+    parser.addoption("--skip-db", action="store_true",
+                     help="skip db tests")
     parser.addoption("--run-high-memory", action="store_true",
                      help="run high memory tests")
     parser.addoption("--only-slow", action="store_true",
@@ -52,6 +54,9 @@ def pytest_runtest_setup(item):
     if 'network' in item.keywords and item.config.getoption("--skip-network"):
         pytest.skip("skipping due to --skip-network")
 
+    if 'db' in item.keywords and item.config.getoption("--skip-db"):
+        pytest.skip("skipping due to --skip-db")
+
     if 'high_memory' in item.keywords and not item.config.getoption(
             "--run-high-memory"):
         pytest.skip(

pandas/tests/io/test_sql.py

@@ -1175,14 +1175,8 @@ class _TestSQLAlchemy(SQLAlchemyMixIn, PandasSQLTest):
     def setup_class(cls):
         cls.setup_import()
         cls.setup_driver()
-
-        # test connection
-        try:
-            conn = cls.connect()
-            conn.connect()
-        except sqlalchemy.exc.OperationalError:
-            msg = "{0} - can't connect to {1} server".format(cls, cls.flavor)
-            pytest.skip(msg)
+        conn = cls.connect()
+        conn.connect()
 
     def load_test_data_and_sql(self):
         self._load_raw_sql()
@@ -1820,13 +1814,10 @@ def connect(cls):
 
     @classmethod
     def setup_driver(cls):
-        try:
-            import pymysql  # noqa
-            cls.driver = 'pymysql'
-            from pymysql.constants import CLIENT
-            cls.connect_args = {'client_flag': CLIENT.MULTI_STATEMENTS}
-        except ImportError:
-            pytest.skip('pymysql not installed')
+        pymysql = pytest.importorskip('pymysql')
+        cls.driver = 'pymysql'
+        cls.connect_args = {
+            'client_flag': pymysql.constants.CLIENT.MULTI_STATEMENTS}
 
     def test_default_type_conversion(self):
         df = sql.read_sql_table("types_test_data", self.conn)
@@ -1889,11 +1880,8 @@ def connect(cls):
 
     @classmethod
     def setup_driver(cls):
-        try:
-            import psycopg2  # noqa
-            cls.driver = 'psycopg2'
-        except ImportError:
-            pytest.skip('psycopg2 not installed')
+        pytest.importorskip('psycopg2')
+        cls.driver = 'psycopg2'
 
     def test_schema_support(self):
         # only test this for postgresql (schema's not supported in
@@ -1991,21 +1979,25 @@ def psql_insert_copy(table, conn, keys, data_iter):
 
 
 @pytest.mark.single
+@pytest.mark.db
 class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy):
     pass
 
 
 @pytest.mark.single
+@pytest.mark.db
 class TestMySQLAlchemyConn(_TestMySQLAlchemy, _TestSQLAlchemyConn):
     pass
 
 
 @pytest.mark.single
+@pytest.mark.db
 class TestPostgreSQLAlchemy(_TestPostgreSQLAlchemy, _TestSQLAlchemy):
     pass
 
 
 @pytest.mark.single
+@pytest.mark.db
 class TestPostgreSQLAlchemyConn(_TestPostgreSQLAlchemy, _TestSQLAlchemyConn):
     pass
 
@@ -2248,13 +2240,6 @@ def tquery(query, con=None, cur=None):
         return list(res)
 
 
-def _skip_if_no_pymysql():
-    try:
-        import pymysql  # noqa
-    except ImportError:
-        pytest.skip('pymysql not installed, skipping')
-
-
 @pytest.mark.single
 class TestXSQLite(SQLiteMixIn):
 
@@ -2463,76 +2448,56 @@ def clean_up(test_table_to_drop):
 
 
 @pytest.mark.single
+@pytest.mark.db
 @pytest.mark.skip(reason="gh-13611: there is no support for MySQL "
                   "if SQLAlchemy is not installed")
 class TestXMySQL(MySQLMixIn):
 
     @pytest.fixture(autouse=True, scope='class')
     def setup_class(cls):
-        _skip_if_no_pymysql()
-
-        # test connection
-        import pymysql
-        try:
-            # Try Travis defaults.
-            # No real user should allow root access with a blank password.
-            pymysql.connect(host='localhost', user='root', passwd='',
-                            db='pandas_nosetest')
-        except pymysql.Error:
-            pass
-        else:
-            return
+        pymysql = pytest.importorskip('pymysql')
+        pymysql.connect(host='localhost', user='root', passwd='',
+                        db='pandas_nosetest')
         try:
             pymysql.connect(read_default_group='pandas')
         except pymysql.ProgrammingError:
-            pytest.skip(
+            raise RuntimeError(
                 "Create a group of connection parameters under the heading "
                 "[pandas] in your system's mysql default file, "
-                "typically located at ~/.my.cnf or /etc/.my.cnf. ")
+                "typically located at ~/.my.cnf or /etc/.my.cnf.")
         except pymysql.Error:
-            pytest.skip(
+            raise RuntimeError(
                 "Cannot connect to database. "
                 "Create a group of connection parameters under the heading "
                 "[pandas] in your system's mysql default file, "
-                "typically located at ~/.my.cnf or /etc/.my.cnf. ")
+                "typically located at ~/.my.cnf or /etc/.my.cnf.")
 
     @pytest.fixture(autouse=True)
     def setup_method(self, request, datapath):
-        _skip_if_no_pymysql()
-        import pymysql
+        pymysql = pytest.importorskip('pymysql')
+        pymysql.connect(host='localhost', user='root', passwd='',
+                        db='pandas_nosetest')
         try:
-            # Try Travis defaults.
-            # No real user should allow root access with a blank password.
-            self.conn = pymysql.connect(host='localhost', user='root',
-                                        passwd='', db='pandas_nosetest')
-        except pymysql.Error:
-            pass
-        else:
-            return
-        try:
-            self.conn = pymysql.connect(read_default_group='pandas')
+            pymysql.connect(read_default_group='pandas')
         except pymysql.ProgrammingError:
-            pytest.skip(
+            raise RuntimeError(
                 "Create a group of connection parameters under the heading "
                 "[pandas] in your system's mysql default file, "
-                "typically located at ~/.my.cnf or /etc/.my.cnf. ")
+                "typically located at ~/.my.cnf or /etc/.my.cnf.")
         except pymysql.Error:
-            pytest.skip(
+            raise RuntimeError(
                 "Cannot connect to database. "
                 "Create a group of connection parameters under the heading "
                 "[pandas] in your system's mysql default file, "
-                "typically located at ~/.my.cnf or /etc/.my.cnf. ")
+                "typically located at ~/.my.cnf or /etc/.my.cnf.")
 
         self.method = request.function
 
     def test_basic(self):
-        _skip_if_no_pymysql()
         frame = tm.makeTimeDataFrame()
         self._check_roundtrip(frame)
 
     def test_write_row_by_row(self):
-
-        _skip_if_no_pymysql()
         frame = tm.makeTimeDataFrame()
         frame.iloc[0, 0] = np.nan
         drop_sql = "DROP TABLE IF EXISTS test"
@@ -2552,7 +2517,6 @@ def test_write_row_by_row(self):
         tm.assert_frame_equal(result, frame, check_less_precise=True)
 
     def test_chunksize_read_type(self):
-        _skip_if_no_pymysql()
         frame = tm.makeTimeDataFrame()
         frame.index.name = "index"
         drop_sql = "DROP TABLE IF EXISTS test"
@@ -2567,7 +2531,6 @@ def test_chunksize_read_type(self):
         tm.assert_frame_equal(frame[:chunksize], chunk_df)
 
     def test_execute(self):
-        _skip_if_no_pymysql()
         frame = tm.makeTimeDataFrame()
         drop_sql = "DROP TABLE IF EXISTS test"
         create_sql = sql.get_schema(frame, 'test')
@@ -2587,7 +2550,6 @@ def test_execute(self):
         tm.assert_frame_equal(result, frame[:1])
 
     def test_schema(self):
-        _skip_if_no_pymysql()
         frame = tm.makeTimeDataFrame()
         create_sql = sql.get_schema(frame, 'test')
         lines = create_sql.splitlines()
@@ -2607,7 +2569,6 @@ def test_schema(self):
 
     @tm.capture_stdout
     def test_execute_fail(self):
-        _skip_if_no_pymysql()
         drop_sql = "DROP TABLE IF EXISTS test"
         create_sql = """
         CREATE TABLE test
@@ -2629,7 +2590,6 @@ def test_execute_fail(self):
             sql.execute('INSERT INTO test VALUES("foo", "bar", 7)', self.conn)
 
     def test_execute_closed_connection(self, request, datapath):
-        _skip_if_no_pymysql()
         drop_sql = "DROP TABLE IF EXISTS test"
         create_sql = """
         CREATE TABLE test
@@ -2654,11 +2614,9 @@ def test_execute_closed_connection(self, request, datapath):
         self.setup_method(request, datapath)
 
     def test_na_roundtrip(self):
-        _skip_if_no_pymysql()
         pass
 
     def _check_roundtrip(self, frame):
-        _skip_if_no_pymysql()
         drop_sql = "DROP TABLE IF EXISTS test_table"
         cur = self.conn.cursor()
         with warnings.catch_warnings():
@@ -2695,13 +2653,11 @@ def _check_roundtrip(self, frame):
         tm.assert_frame_equal(expected, result)
 
     def test_keyword_as_column_names(self):
-        _skip_if_no_pymysql()
         df = DataFrame({'From': np.ones(5)})
         sql.to_sql(df, con=self.conn, name='testkeywords',
                    if_exists='replace', index=False)
 
     def test_if_exists(self):
-        _skip_if_no_pymysql()
         df_if_exists_1 = DataFrame({'col1': [1, 2], 'col2': ['A', 'B']})
         df_if_exists_2 = DataFrame(
             {'col1': [3, 4, 5], 'col2': ['C', 'D', 'E']})

pandas/util/_tester.py

@@ -16,7 +16,7 @@ def test(extra_args=None):
         import hypothesis  # noqa
     except ImportError:
         raise ImportError("Need hypothesis>=3.58 to run tests")
-    cmd = ['--skip-slow', '--skip-network']
+    cmd = ['--skip-slow', '--skip-network', '--skip-db']
     if extra_args:
         if not isinstance(extra_args, list):
             extra_args = [extra_args]

setup.cfg

@@ -67,6 +67,7 @@ markers =
     single: mark a test as single cpu only
     slow: mark a test as slow
     network: mark a test as network
+    db: tests requiring a database (mysql or postgres)
     high_memory: mark a test as a high-memory only
     clipboard: mark a pd.read_clipboard test
 doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL