From ac530e2b860693cbb62bbd596a8724d6400d1f27 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 15:11:36 -0400 Subject: [PATCH 01/13] test on python3.7 --- ...uirements-3.6-MASTER.pip => requirements-3.7-MASTER.pip} | 0 nox.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename ci/{requirements-3.6-MASTER.pip => requirements-3.7-MASTER.pip} (100%) diff --git a/ci/requirements-3.6-MASTER.pip b/ci/requirements-3.7-MASTER.pip similarity index 100% rename from ci/requirements-3.6-MASTER.pip rename to ci/requirements-3.7-MASTER.pip diff --git a/nox.py b/nox.py index 53f3b8a6..1ab7a371 100644 --- a/nox.py +++ b/nox.py @@ -80,8 +80,8 @@ def test36(session): @nox.session -def test36master(session): - session.interpreter = 'python3.6' +def test37master(session): + session.interpreter = 'python3.7' session.install( '--pre', '--upgrade', @@ -89,7 +89,7 @@ def test36master(session): '-f', PANDAS_PRE_WHEELS, 'pandas') session.install( - '-r', os.path.join('.', 'ci', 'requirements-3.6-MASTER.pip')) + '-r', os.path.join('.', 'ci', 'requirements-3.7-MASTER.pip')) default(session) From a07e92ec6d3c08889087d9caaab5410599ec83af Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 15:13:02 -0400 Subject: [PATCH 02/13] skip conda tests --- nox.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nox.py b/nox.py index 1ab7a371..e2762992 100644 --- a/nox.py +++ b/nox.py @@ -70,9 +70,12 @@ def test35(session): '-r', os.path.join('.', 'ci', 'requirements-3.5-0.18.1.pip')) default(session) +# TODO: fix conda issues around namespace packages +# @nox.session +# def test36(session): -@nox.session -def test36(session): + +def _test36(session): session.interpreter = 'python3.6' session.install( '-r', os.path.join('.', 'ci', 'requirements-3.6-0.20.1.conda')) From 76d12ce3aca8f2bd873a8164c831c04ab8f2749b Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 15:21:37 -0400 Subject: [PATCH 03/13] skip in the correct place --- .travis.yml | 7 ++++--- nox.py | 6 +----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1ab8f9c..5f20cfe5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,10 @@ matrix: - os: linux python: 3.5 env: PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' - - os: linux - python: 3.6 - env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' + # TODO: fix conda issues around namespace packages + # - os: linux + # python: 3.6 + # env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' - os: linux python: 3.6 env: PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' diff --git a/nox.py b/nox.py index e2762992..8a1646dc 100644 --- a/nox.py +++ b/nox.py @@ -70,12 +70,8 @@ def test35(session): '-r', os.path.join('.', 'ci', 'requirements-3.5-0.18.1.pip')) default(session) -# TODO: fix conda issues around namespace packages -# @nox.session -# def test36(session): - -def _test36(session): +def test36(session): session.interpreter = 'python3.6' session.install( '-r', os.path.join('.', 'ci', 'requirements-3.6-0.20.1.conda')) From cfad6cf307f7e0df4624987184e19c8ee4eddb2d Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 15:27:36 -0400 Subject: [PATCH 04/13] another rename --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f20cfe5..450c0e3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,8 @@ matrix: # python: 3.6 # env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' - os: linux - python: 3.6 - env: PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' + python: 3.7 + env: PYTHON=3.7 PANDAS=MASTER COVERAGE='false' LINT='true' env: before_install: @@ -51,7 +51,7 @@ install: script: - if [[ $PYTHON == '2.7' ]]; then nox -s test27 ; fi - if [[ $PYTHON == '3.5' ]]; then nox -s test35 ; fi - - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test36master ; fi + - if [[ $PYTHON == '3.7' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test37master ; fi - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; if [ -f "$REQ.conda" ]; then pytest --quiet -m 'not local_auth' -v tests ; From c9a381dff6afcfa9c07cc37fc4ce00dad07dc8e6 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 16:20:30 -0400 Subject: [PATCH 05/13] run on xenial on travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 450c0e3d..2fcfd64f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ language: python +dist: xenial +sudo: true matrix: include: From e467bfaba11376b8918ba5016f42730c6bc92bf6 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 16:20:52 -0400 Subject: [PATCH 06/13] try with no sudo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2fcfd64f..8e2e4e9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python dist: xenial -sudo: true +# sudo: true matrix: include: From f512ac2857af9597e550718ecd2fc87c8f33566c Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 16:23:00 -0400 Subject: [PATCH 07/13] move dist to correct location --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e2e4e9e..46614070 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,4 @@ language: python -dist: xenial -# sudo: true matrix: include: @@ -16,6 +14,8 @@ matrix: # env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' - os: linux python: 3.7 + dist: xenial + sudo: true env: PYTHON=3.7 PANDAS=MASTER COVERAGE='false' LINT='true' env: From 555d8d2d4d1def3a028a542435f2068e59161a84 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 16:27:18 -0400 Subject: [PATCH 08/13] link to GH issue --- nox.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nox.py b/nox.py index 42bfead9..6c953b77 100644 --- a/nox.py +++ b/nox.py @@ -19,7 +19,8 @@ def default(session): session.install("mock", "pytest", "pytest-cov") session.install("-e", ".") - # Skip local auth tests on Travis. + # Skip local auth tests on Travis + # ref https://github.com/pydata/pandas-gbq/issues/189 additional_args = list(session.posargs) if "TRAVIS_BUILD_DIR" in os.environ: additional_args = additional_args + ["-m", "not local_auth"] From c26812930250b34281434a3db02c2a89e59d705d Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 16:58:38 -0400 Subject: [PATCH 09/13] try no sudo --- .travis.yml | 2 +- nox.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 46614070..92fa36d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: - os: linux python: 3.7 dist: xenial - sudo: true + # sudo: true env: PYTHON=3.7 PANDAS=MASTER COVERAGE='false' LINT='true' env: diff --git a/nox.py b/nox.py index 6c953b77..2ab1314f 100644 --- a/nox.py +++ b/nox.py @@ -23,6 +23,7 @@ def default(session): # ref https://github.com/pydata/pandas-gbq/issues/189 additional_args = list(session.posargs) if "TRAVIS_BUILD_DIR" in os.environ: + additional_args = additional_args + ["-m", "not local_auth"] session.run( @@ -81,12 +82,13 @@ def test36(session): @nox.session def test37master(session): - session.interpreter = 'python3.7' + session.interpreter = "python3.7" session.install( "--pre", "--upgrade", "--timeout=60", "-f", PANDAS_PRE_WHEELS, "pandas" ) session.install( - '-r', os.path.join('.', 'ci', 'requirements-3.7-MASTER.pip')) + "-r", os.path.join(".", "ci", "requirements-3.7-MASTER.pip") + ) default(session) From 23b6837f92baf387a154a2d0b970611a837686fb Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 17:09:51 -0400 Subject: [PATCH 10/13] requires sudo (but then auth doesn't work...) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 92fa36d7..46614070 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: - os: linux python: 3.7 dist: xenial - # sudo: true + sudo: true env: PYTHON=3.7 PANDAS=MASTER COVERAGE='false' LINT='true' env: From 60e8e85abea110f87201c85b6e9bb5ec7b591028 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 21:25:08 -0400 Subject: [PATCH 11/13] test to see if it's sudo --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 46614070..aca5818b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ matrix: env: PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' - os: linux python: 3.5 + sudo: true env: PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' # TODO: fix conda issues around namespace packages # - os: linux From a423a1b2da578a1b3f1c6fe69568318c02a15424 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 30 Aug 2018 22:05:47 -0400 Subject: [PATCH 12/13] Revert "test to see if it's sudo" This reverts commit 60e8e85abea110f87201c85b6e9bb5ec7b591028. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aca5818b..46614070 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ matrix: env: PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' - os: linux python: 3.5 - sudo: true env: PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' # TODO: fix conda issues around namespace packages # - os: linux From bbed7ecf135c52b0514f39841c68e3b90d7e4bc4 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Fri, 31 Aug 2018 23:49:22 -0400 Subject: [PATCH 13/13] Merge --- .travis.yml | 1 + docs/source/changelog.rst | 1 + pandas_gbq/auth.py | 27 ++++++++++++++++++++------- pandas_gbq/gbq.py | 5 +++++ tests/system/test_auth.py | 25 +++++++++++++++++++------ tests/unit/test_auth.py | 4 +++- tests/unit/test_gbq.py | 14 ++++++++++++++ 7 files changed, 63 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 46614070..9de96ad4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ matrix: python: 3.5 env: PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' # TODO: fix conda issues around namespace packages + # https://github.com/pydata/pandas-gbq/issues/189 # - os: linux # python: 3.6 # env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index b4b53488..8681de27 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -12,6 +12,7 @@ Changelog (:issue:`128`) - Reduced verbosity of logging from ``read_gbq``, particularly for short queries. (:issue:`201`) +- Avoid ``SELECT 1`` query when running ``to_gbq``. (:issue:`202`) .. _changelog-0.6.0: diff --git a/pandas_gbq/auth.py b/pandas_gbq/auth.py index c27d342e..2bc2efea 100644 --- a/pandas_gbq/auth.py +++ b/pandas_gbq/auth.py @@ -16,20 +16,28 @@ def get_credentials( - private_key=None, project_id=None, reauth=False, auth_local_webserver=False + private_key=None, + project_id=None, + reauth=False, + auth_local_webserver=False, + try_credentials=None, ): + if try_credentials is None: + try_credentials = _try_credentials + if private_key: return get_service_account_credentials(private_key) # Try to retrieve Application Default Credentials credentials, default_project = get_application_default_credentials( - project_id=project_id + try_credentials, project_id=project_id ) if credentials: return credentials, default_project credentials = get_user_account_credentials( + try_credentials, project_id=project_id, reauth=reauth, auth_local_webserver=auth_local_webserver, @@ -79,7 +87,7 @@ def get_service_account_credentials(private_key): ) -def get_application_default_credentials(project_id=None): +def get_application_default_credentials(try_credentials, project_id=None): """ This method tries to retrieve the "default application credentials". This could be useful for running code on Google Cloud Platform. @@ -111,10 +119,11 @@ def get_application_default_credentials(project_id=None): # used with BigQuery. For example, we could be running on a GCE instance # that does not allow the BigQuery scopes. billing_project = project_id or default_project - return _try_credentials(billing_project, credentials), billing_project + return try_credentials(billing_project, credentials), billing_project def get_user_account_credentials( + try_credentials, project_id=None, reauth=False, auth_local_webserver=False, @@ -151,7 +160,9 @@ def get_user_account_credentials( os.rename("bigquery_credentials.dat", credentials_path) credentials = load_user_account_credentials( - project_id=project_id, credentials_path=credentials_path + try_credentials, + project_id=project_id, + credentials_path=credentials_path, ) client_config = { @@ -187,7 +198,9 @@ def get_user_account_credentials( return credentials -def load_user_account_credentials(project_id=None, credentials_path=None): +def load_user_account_credentials( + try_credentials, project_id=None, credentials_path=None +): """ Loads user account credentials from a local file. @@ -230,7 +243,7 @@ def load_user_account_credentials(project_id=None, credentials_path=None): request = google.auth.transport.requests.Request() credentials.refresh(request) - return _try_credentials(project_id, credentials) + return try_credentials(project_id, credentials) def get_default_credentials_path(): diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index eba08009..c45384e4 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -171,6 +171,7 @@ def __init__( auth_local_webserver=False, dialect="legacy", location=None, + try_credentials=None, ): from google.api_core.exceptions import GoogleAPIError from google.api_core.exceptions import ClientError @@ -189,6 +190,7 @@ def __init__( project_id=project_id, reauth=reauth, auth_local_webserver=auth_local_webserver, + try_credentials=try_credentials, ) if self.project_id is None: @@ -804,6 +806,9 @@ def to_gbq( private_key=private_key, auth_local_webserver=auth_local_webserver, location=location, + # Avoid reads when writing tables. + # https://github.com/pydata/pandas-gbq/issues/202 + try_credentials=lambda project, creds: creds, ) dataset_id, table_id = destination_table.rsplit(".", 1) diff --git a/tests/system/test_auth.py b/tests/system/test_auth.py index 7ccc79c2..f7cdf014 100644 --- a/tests/system/test_auth.py +++ b/tests/system/test_auth.py @@ -66,9 +66,13 @@ def test_get_application_default_credentials_does_not_throw_error(): with mock.patch( "google.auth.default", side_effect=DefaultCredentialsError() ): - credentials, _ = auth.get_application_default_credentials() + credentials, _ = auth.get_application_default_credentials( + try_credentials=auth._try_credentials + ) else: - credentials, _ = auth.get_application_default_credentials() + credentials, _ = auth.get_application_default_credentials( + try_credentials=auth._try_credentials + ) assert credentials is None @@ -77,7 +81,9 @@ def test_get_application_default_credentials_returns_credentials(): pytest.skip("Cannot get default_credentials " "from the environment!") from google.auth.credentials import Credentials - credentials, default_project = auth.get_application_default_credentials() + credentials, default_project = auth.get_application_default_credentials( + try_credentials=auth._try_credentials + ) assert isinstance(credentials, Credentials) assert default_project is not None @@ -88,7 +94,9 @@ def test_get_user_account_credentials_bad_file_returns_credentials(): from google.auth.credentials import Credentials with mock.patch("__main__.open", side_effect=IOError()): - credentials = auth.get_user_account_credentials() + credentials = auth.get_user_account_credentials( + try_credentials=auth._try_credentials + ) assert isinstance(credentials, Credentials) @@ -97,7 +105,9 @@ def test_get_user_account_credentials_returns_credentials(project_id): from google.auth.credentials import Credentials credentials = auth.get_user_account_credentials( - project_id=project_id, auth_local_webserver=True + project_id=project_id, + auth_local_webserver=True, + try_credentials=auth._try_credentials, ) assert isinstance(credentials, Credentials) @@ -107,6 +117,9 @@ def test_get_user_account_credentials_reauth_returns_credentials(project_id): from google.auth.credentials import Credentials credentials = auth.get_user_account_credentials( - project_id=project_id, auth_local_webserver=True, reauth=True + project_id=project_id, + auth_local_webserver=True, + reauth=True, + try_credentials=auth._try_credentials, ) assert isinstance(credentials, Credentials) diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py index 50c71b27..d8107a40 100644 --- a/tests/unit/test_auth.py +++ b/tests/unit/test_auth.py @@ -95,7 +95,9 @@ def mock_default_credentials(scopes=None, request=None): google.auth.credentials.Credentials ) - def mock_load_credentials(project_id=None, credentials_path=None): + def mock_load_credentials( + try_credentials, project_id=None, credentials_path=None + ): return mock_user_credentials monkeypatch.setattr( diff --git a/tests/unit/test_gbq.py b/tests/unit/test_gbq.py index df80d5d2..4a42e057 100644 --- a/tests/unit/test_gbq.py +++ b/tests/unit/test_gbq.py @@ -46,6 +46,7 @@ def mock_bigquery_client(monkeypatch): # Mock table creation. mock_client.get_table.side_effect = NotFound("nope") monkeypatch.setattr(gbq.GbqConnector, "get_client", lambda _: mock_client) + return mock_client def mock_none_credentials(*args, **kwargs): @@ -210,6 +211,19 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): assert len(recwarn) == 0 +def test_to_gbq_doesnt_run_query( + recwarn, mock_bigquery_client, min_bq_version +): + try: + gbq.to_gbq( + DataFrame([[1]]), "dataset.tablename", project_id="my-project" + ) + except gbq.TableCreationError: + pass + + mock_bigquery_client.query.assert_not_called() + + def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): from pandas_gbq import auth