TST: Use nox docker image for PIP CircleCI tests (#251)

tswast · web-flow · commit f729a44a4874 · 2019-02-15T15:37:12.000-08:00
* TST: Use nox docker image for PIP CircleCI tests

Fixes the CircleCI build.
Clean up the noxfile to use the same requirements files as CI.

Temporarily disables installing pandas nightlies and
google-cloud-bigquery from master since they were taking several minutes
to install in my local test environment.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,48 +1,48 @@
 version: 2
 jobs:
   # Pip
-  "pip-2.7-0.19.2":
+  "pip-2.7":
     docker:
-      - image: python:2.7
-    environment:
-      PYTHON: "2.7"
-      PANDAS: "0.19.2"
+      - image: thekevjames/nox
     steps:
       - checkout
       - run: ci/config_auth.sh
-      - run: ci/run_pip.sh
-  "pip-3.5-0.18.1":
+      - run: nox -s unit-2.7 system-2.7
+
+  "pip-3.5":
     docker:
-      - image: python:3.5
-    environment:
-      PYTHON: "3.5"
-      PANDAS: "0.18.1"
+      - image: thekevjames/nox
     steps:
       - checkout
       - run: ci/config_auth.sh
-      - run: ci/run_pip.sh
-  "pip-3.6-MASTER":
+      - run: nox -s unit-3.5 system-3.5
+
+  "pip-3.6":
     docker:
-      - image: python:3.6
-    environment:
-      PYTHON: "3.6"
-      PANDAS: "MASTER"
+      - image: thekevjames/nox
     steps:
       - checkout
       - run: ci/config_auth.sh
-      - run: ci/run_pip.sh
-      # Coverage
-      - run: codecov
-  "pip-3.7-0.23.4":
+      - run: nox -s unit-3.6 system-3.6
+
+  "pip-3.7":
     docker:
-      - image: python:3.7
-    environment:
-      PYTHON: "3.7"
-      PANDAS: "0.23.4"
+      - image: thekevjames/nox
     steps:
       - checkout
       - run: ci/config_auth.sh
-      - run: ci/run_pip.sh
+      - run: nox -s unit-3.7 system-3.7 cover
+
+  "lint":
+    docker:
+      - image: thekevjames/nox
+    environment:
+      # Resolve "Python 3 was configured to use ASCII as encoding for the environment"
+      LC_ALL: C.UTF-8
+      LANG: C.UTF-8
+    steps:
+      - checkout
+      - run: nox -s lint
 
   # Conda
   "conda-3.6-0.20.1":
@@ -56,20 +56,13 @@ jobs:
       - run: ci/config_auth.sh
       - run: ci/run_conda.sh
 
-  lint:
-    docker:
-      - image: python:3.6
-    steps:
-      - checkout
-      - run: pip install nox
-      - run: nox -s lint
 workflows:
   version: 2
   build:
     jobs:
-      - "pip-2.7-0.19.2"
-      - "pip-3.5-0.18.1"
-      - "pip-3.6-MASTER"
-      - "pip-3.7-0.23.4"
-      - "conda-3.6-0.20.1"
-      - lint
+      - "pip-2.7"
+      - "pip-3.5"
+      - "pip-3.6"
+      - "pip-3.7"
+      - lint
+      - "conda-3.6-0.20.1"
diff --git a/.flake8 b/.flake8
@@ -0,0 +1,8 @@
+[flake8]
+ignore = E203, E266, E501, W503
+exclude =
+  # Standard linting exemptions.
+  __pycache__,
+  .git,
+  *.pyc,
+  conf.py
diff --git a/.gitignore b/.gitignore
@@ -68,6 +68,8 @@ dist
 # wheel files
 *.whl
 **/wheelhouse/*
+pip-wheel-metadata
+
 # coverage
 .coverage
 .testmondata
diff --git a/ci/requirements-2.7-0.19.2.pip b/ci/requirements-2.7-0.19.2.pip
diff --git a/ci/requirements-2.7.pip b/ci/requirements-2.7.pip
@@ -0,0 +1,6 @@
+mock
+pandas==0.17.1
+google-auth==1.4.1
+google-auth-oauthlib==0.0.1
+google-cloud-bigquery==0.32.0
+pydata-google-auth==0.1.2
diff --git a/ci/requirements-3.5-0.18.1.pip b/ci/requirements-3.5-0.18.1.pip
diff --git a/ci/requirements-3.5.pip b/ci/requirements-3.5.pip
@@ -0,0 +1,5 @@
+pandas==0.19.0
+google-auth==1.4.1
+google-auth-oauthlib==0.0.1
+google-cloud-bigquery==0.32.0
+pydata-google-auth==0.1.2
diff --git a/ci/requirements-3.6-MASTER.pip b/ci/requirements-3.6-MASTER.pip
@@ -1,3 +1,5 @@
+--pre -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com/ pandas
 git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core
 git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=core
 git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery
+pydata-google-auth==0.1.2
diff --git a/ci/requirements-3.6.pip b/ci/requirements-3.6.pip
@@ -0,0 +1,3 @@
+pandas
+pydata-google-auth
+google-cloud-bigquery
diff --git a/ci/requirements-3.7-0.23.4.pip b/ci/requirements-3.7-0.23.4.pip
diff --git a/ci/requirements-3.7.pip b/ci/requirements-3.7.pip
@@ -0,0 +1,3 @@
+pandas==0.24.0
+google-cloud-bigquery==1.9.0
+pydata-google-auth==0.1.2
diff --git a/noxfile.py b/noxfile.py
@@ -8,45 +8,34 @@
 
 import nox
 
-PANDAS_PRE_WHEELS = (
-    "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83"
-    ".ssl.cf2.rackcdn.com"
-)
-
 
+supported_pythons = ["2.7", "3.5", "3.6", "3.7"]
 latest_python = "3.6"
 
 
 @nox.session
-def test(session):
-    session.install("mock", "pytest", "pytest-cov")
+def lint(session, python=latest_python):
+    session.install("black", "flake8")
     session.install("-e", ".")
+    session.run("flake8", "pandas_gbq")
+    session.run("flake8", "tests")
+    session.run("black", "--check", ".")
 
-    # Skip local auth tests on Travis.
-    additional_args = list(session.posargs)
-    if "TRAVIS_BUILD_DIR" in os.environ:
-        additional_args = additional_args + ["-m", "not local_auth"]
 
-    session.run(
-        "pytest",
-        os.path.join(".", "tests"),
-        "--quiet",
-        "--cov=pandas_gbq",
-        "--cov=tests.unit",
-        "--cov-report",
-        "xml:/tmp/pytest-cov.xml",
-        *additional_args
-    )
+@nox.session(python=latest_python)
+def blacken(session):
+    session.install("black")
+    session.run("black", ".")
 
 
-@nox.session
+@nox.session(python=supported_pythons)
 def unit(session):
     session.install("mock", "pytest", "pytest-cov")
     session.install("-e", ".")
     session.run(
         "pytest",
         os.path.join(".", "tests", "unit"),
-        "--quiet",
+        "-v",
         "--cov=pandas_gbq",
         "--cov=tests.unit",
         "--cov-report",
@@ -56,32 +45,32 @@ def unit(session):
 
 
 @nox.session
-def test_earliest_deps(session, python="3.5"):
-    session.install(
-        "-r", os.path.join(".", "ci", "requirements-3.5-0.18.1.pip")
-    )
-    test(session)
+def cover(session, python=latest_python):
+    session.install("coverage", "pytest-cov")
+    session.run("coverage", "report", "--show-missing", "--fail-under=40")
+    session.run("coverage", "erase")
 
 
-@nox.session
-def test_latest_deps(session, python=latest_python):
+@nox.session(python=supported_pythons)
+def system(session):
+    session.install("pytest", "pytest-cov")
     session.install(
-        "--pre", "--upgrade", "--timeout=60", "-f", PANDAS_PRE_WHEELS, "pandas"
+        "-r",
+        os.path.join(".", "ci", "requirements-{}.pip".format(session.python)),
     )
     session.install(
-        "-r", os.path.join(".", "ci", "requirements-3.6-MASTER.pip")
+        "-e",
+        ".",
+        # Use dependencies from requirements file instead.
+        # This enables testing with specific versions of the dependencies.
+        "--no-dependencies",
     )
-    test(session)
-
-
-@nox.session
-def lint(session, python=latest_python):
-    session.install("black")
-    session.run("black", "--check", ".")
 
+    # Skip local auth tests on CI.
+    additional_args = list(session.posargs)
+    if "CIRCLECI" in os.environ:
+        additional_args = additional_args + ["-m", "not local_auth"]
 
-@nox.session
-def cover(session, python=latest_python):
-    session.install("coverage", "pytest-cov")
-    session.run("coverage", "report", "--show-missing", "--fail-under=40")
-    session.run("coverage", "erase")
+    session.run(
+        "pytest", os.path.join(".", "tests", "system"), "-v", *additional_args
+    )
diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py
@@ -1,5 +1,4 @@
 import logging
-import os
 import time
 import warnings
 from collections import OrderedDict
@@ -69,7 +68,7 @@ def _check_google_client_version():
 def _test_google_api_imports():
 
     try:
-        import pydata_google_auth
+        import pydata_google_auth  # noqa
     except ImportError as ex:
         raise ImportError(
             "pandas-gbq requires pydata-google-auth: {0}".format(ex)
diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py
@@ -46,11 +46,11 @@ def test_read_gbq_should_use_dialect(mock_bigquery_client):
     pandas_gbq.read_gbq("SELECT 1")
 
     _, kwargs = mock_bigquery_client.query.call_args
-    assert kwargs["job_config"].use_legacy_sql == True
+    assert kwargs["job_config"].use_legacy_sql
 
     pandas_gbq.context.dialect = "standard"
     pandas_gbq.read_gbq("SELECT 1")
 
     _, kwargs = mock_bigquery_client.query.call_args
-    assert kwargs["job_config"].use_legacy_sql == False
+    assert not kwargs["job_config"].use_legacy_sql
     pandas_gbq.context.dialect = None  # Reset the global state.
diff --git a/tests/unit/test_gbq.py b/tests/unit/test_gbq.py
@@ -58,7 +58,6 @@ def mock_get_user_credentials(*args, **kwargs):
 
 @pytest.fixture(autouse=True)
 def no_auth(monkeypatch):
-    from pandas_gbq import auth
     import pydata_google_auth
 
     monkeypatch.setattr(pydata_google_auth, "default", mock_get_credentials)

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+pandas`
	`2`	`+pydata-google-auth`
	`3`	`+google-cloud-bigquery`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+pandas==0.24.0`
	`2`	`+google-cloud-bigquery==1.9.0`
	`3`	`+pydata-google-auth==0.1.2`