From c85a6850ad318e390067ace7c8eef65ca6965101 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Mon, 19 Feb 2018 21:41:35 -0500 Subject: [PATCH 01/22] project_id is optional --- pandas_gbq/gbq.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 382f276b..8bfc04cf 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -761,7 +761,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, ---------- query : str SQL-Like Query to return data values - project_id : str + project_id : str (optional) Google BigQuery Account project ID. index_col : str (optional) Name of result column to use for index in results DataFrame @@ -815,9 +815,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, _test_google_api_imports() - if not project_id: - raise TypeError("Missing required parameter: project_id") - if dialect not in ('legacy', 'standard'): raise ValueError("'{0}' is not valid for dialect".format(dialect)) @@ -897,7 +894,7 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=None, DataFrame to be written destination_table : string Name of table to be written, in the form 'dataset.tablename' - project_id : str + project_id : str (optional) Google BigQuery Account project ID. chunksize : int (default None) Number of rows to be inserted in each chunk from the dataframe. Use From 17a2ede2486bb6b0e2a7f7968d903fe3310cba58 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 16:56:32 -0400 Subject: [PATCH 02/22] gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 147e7e1e..37e24ee9 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,8 @@ .ipynb_checkpoints .tags .pytest_cache -.testmondata +.testmon* +.vscode/ # Docs # ######## From 86b6fc50b28e9f784762282c61585a12587326e9 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 16:56:59 -0400 Subject: [PATCH 03/22] docstring --- pandas_gbq/gbq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 7b1bc011..45fc6766 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -752,7 +752,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, ---------- query : str SQL-Like Query to return data values - project_id : str (optional) + project_id : str (optional when using credentials / service account) Google BigQuery Account project ID. index_col : str (optional) Name of result column to use for index in results DataFrame @@ -857,7 +857,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, return final_df -def to_gbq(dataframe, destination_table, project_id, chunksize=None, +def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, verbose=None, reauth=False, if_exists='fail', private_key=None, auth_local_webserver=False, table_schema=None): """Write a DataFrame to a Google BigQuery table. @@ -889,7 +889,7 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=None, DataFrame to be written destination_table : string Name of table to be written, in the form 'dataset.tablename' - project_id : str (optional) + project_id : str (optional when using credentials / service account) Google BigQuery Account project ID. chunksize : int (default None) Number of rows to be inserted in each chunk from the dataframe. Use From 11c146e00c6d82b74f349d74fb269f871b183062 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 17:11:18 -0400 Subject: [PATCH 04/22] don't skip if no project --- pandas_gbq/tests/test_gbq.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index 2df1b9bd..e23259f7 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -52,12 +52,9 @@ def _get_dataset_prefix_random(): def _get_project_id(): + return (os.environ.get('GBQ_PROJECT_ID') + or os.environ.get('GOOGLE_CLOUD_PROJECT')) - project = os.environ.get('GBQ_PROJECT_ID') - if not project: - pytest.skip( - "Cannot run integration tests without a project id") - return project def _get_private_key_path(): @@ -87,9 +84,12 @@ def _test_imports(): gbq._test_google_api_imports() -@pytest.fixture -def project(): - return _get_project_id() +@pytest.fixture(params=['env']) +def project(request): + if request.param == 'env': + return _get_project_id() + elif request.param == 'none': + return None def _check_if_can_get_correct_default_credentials(): @@ -310,13 +310,8 @@ def test_to_gbq_should_fail_if_invalid_table_name_passed(self): with pytest.raises(gbq.NotFoundException): gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") - def test_to_gbq_with_no_project_id_given_should_fail(self): - with pytest.raises(TypeError): - gbq.to_gbq(DataFrame(), 'dataset.tablename') - - def test_read_gbq_with_no_project_id_given_should_fail(self): - with pytest.raises(TypeError): - gbq.read_gbq('SELECT 1') + def test_read_gbq_with_no_project_id_given_should_pass(self): + gbq.read_gbq('SELECT 1') def test_that_parse_data_works_properly(self): From 817c63a329aad5621285c1d09a422ba75c1f67ff Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 17:23:53 -0400 Subject: [PATCH 05/22] formatting --- pandas_gbq/tests/test_gbq.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index e23259f7..4e9472db 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -52,9 +52,8 @@ def _get_dataset_prefix_random(): def _get_project_id(): - return (os.environ.get('GBQ_PROJECT_ID') - or os.environ.get('GOOGLE_CLOUD_PROJECT')) - + return (os.environ.get('GBQ_PROJECT_ID') + or os.environ.get('GOOGLE_CLOUD_PROJECT')) # noqa def _get_private_key_path(): From f0a8d360178847229958d3cca50b31ca44447f45 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 20:34:40 -0400 Subject: [PATCH 06/22] add marks to allow test selecting --- pandas_gbq/tests/test_gbq.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index 4e9472db..18744377 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -172,7 +172,11 @@ def test_generate_bq_schema_deprecated(): gbq.generate_bq_schema(df) -@pytest.fixture(params=['local', 'service_path', 'service_creds']) +@pytest.fixture(params=[ + pytest.param('local', marks=pytest.mark.local_auth), + pytest.param('service_path', marks=pytest.mark.s_path_auth), + pytest.param('service_creds', marks=pytest.mark.s_cred_auth), +]) def auth_type(request): auth = request.param @@ -237,6 +241,7 @@ def test_should_be_able_to_get_results_from_query(self, gbq_connector): assert pages is not None +@pytest.mark.local_auth class TestGBQConnectorIntegrationWithLocalUserAccountAuth(object): @pytest.fixture(autouse=True) From 4ecfcfc33854517e575bdd8b5f94ade044482823 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 21:00:48 -0400 Subject: [PATCH 07/22] assign mark to test --- pandas_gbq/tests/test_gbq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index 18744377..f89b3c40 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -314,8 +314,8 @@ def test_to_gbq_should_fail_if_invalid_table_name_passed(self): with pytest.raises(gbq.NotFoundException): gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") - def test_read_gbq_with_no_project_id_given_should_pass(self): - gbq.read_gbq('SELECT 1') + def test_read_gbq_with_no_project_id_given_should_pass(self, credentials): + gbq.read_gbq('SELECT 1', private_key=credentials) def test_that_parse_data_works_properly(self): From c402952716b21229fec02caba589135ac89fe02a Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 21:12:38 -0400 Subject: [PATCH 08/22] explicitly chose with auth to do from travis --- .travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 92129cc6..b5fc587c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,10 @@ sudo: false language: python env: - - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' - - PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' - - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' - - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' + - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' AUTH='s_path' + - PYTHON=3.5 PANDAS=0.22.0 COVERAGE='true' LINT='false' AUTH='s_path' + - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' AUTH='s_cred' + - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' AUTH='s_cred' before_install: - echo "before_install" @@ -42,6 +42,7 @@ install: - python setup.py install script: - - pytest -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq + - if [[ $AUTH == 's_path' ]]; then pytest -m 'not local_auth' -m 'not s_cred_auth' -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ; fi + - if [[ $AUTH == 's_cred' ]]; then pytest -m 'not local_auth' -m 'not s_path_auth' -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ; fi - if [[ $COVERAGE == 'true' ]]; then codecov ; fi - if [[ $LINT == 'true' ]]; then flake8 pandas_gbq -v ; fi From 476bcf1aed7c09a0aff2e35953e38c0252f25421 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 21:25:39 -0400 Subject: [PATCH 09/22] too hasty to change the pandas installation --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b5fc587c..b497f803 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ language: python env: - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' AUTH='s_path' - - PYTHON=3.5 PANDAS=0.22.0 COVERAGE='true' LINT='false' AUTH='s_path' + - PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' AUTH='s_path' - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' AUTH='s_cred' - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' AUTH='s_cred' From ef59bca98ecd246164c2daf8520443de5ebcc4e9 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 21:43:22 -0400 Subject: [PATCH 10/22] do what needs doing --- pandas_gbq/gbq.py | 3 +++ pandas_gbq/tests/test_gbq.py | 14 ++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 45fc6766..48de98c4 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -10,6 +10,7 @@ import numpy as np from pandas import DataFrame, compat from pandas.compat import lzip +import google logger = logging.getLogger(__name__) @@ -170,6 +171,8 @@ def __init__(self, project_id, reauth=False, from google.api_core.exceptions import GoogleAPIError from google.api_core.exceptions import ClientError self.http_error = (ClientError, GoogleAPIError) + if not project_id: + _, project_id = google.auth.default() self.project_id = project_id self.reauth = reauth self.private_key = private_key diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index f89b3c40..f2cb8732 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -100,11 +100,12 @@ def _check_if_can_get_correct_default_credentials(): from google.auth.exceptions import DefaultCredentialsError try: - credentials, _ = google.auth.default(scopes=[gbq.GbqConnector.scope]) + credentials, project = google.auth.default(scopes=[gbq.GbqConnector.scope]) except (DefaultCredentialsError, IOError): return False - return gbq._try_credentials(_get_project_id(), credentials) is not None + return gbq._try_credentials( + project or _get_project_id(), credentials) is not None def clean_gbq_environment(dataset_prefix, private_key=None): @@ -182,15 +183,8 @@ def auth_type(request): auth = request.param if auth == 'local': - - if _in_travis_environment(): - pytest.skip("Cannot run local auth in travis environment") - + pass elif auth == 'service_path': - - if _in_travis_environment(): - pytest.skip("Only run one auth type in Travis to save time") - _skip_if_no_private_key_path() elif auth == 'service_creds': _skip_if_no_private_key_contents() From 2112acd31033ceb909d7ad168972d3636f6e832d Mon Sep 17 00:00:00 2001 From: stickler-ci Date: Sun, 1 Apr 2018 01:43:06 +0000 Subject: [PATCH 11/22] Fixing style errors. --- pandas_gbq/gbq.py | 2 +- pandas_gbq/tests/test_gbq.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 48de98c4..8c2077a6 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -172,7 +172,7 @@ def __init__(self, project_id, reauth=False, from google.api_core.exceptions import ClientError self.http_error = (ClientError, GoogleAPIError) if not project_id: - _, project_id = google.auth.default() + _, project_id = google.auth.default() self.project_id = project_id self.reauth = reauth self.private_key = private_key diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index f2cb8732..8f529184 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -100,7 +100,8 @@ def _check_if_can_get_correct_default_credentials(): from google.auth.exceptions import DefaultCredentialsError try: - credentials, project = google.auth.default(scopes=[gbq.GbqConnector.scope]) + credentials, project = google.auth.default( + scopes=[gbq.GbqConnector.scope]) except (DefaultCredentialsError, IOError): return False From 6cc99f6ddf26cadf6f4a64f52bd65b910f45efab Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 21:49:31 -0400 Subject: [PATCH 12/22] docstring & import order --- pandas_gbq/gbq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 8c2077a6..41a30df2 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -7,10 +7,10 @@ from distutils.version import StrictVersion from time import sleep +import google import numpy as np from pandas import DataFrame, compat from pandas.compat import lzip -import google logger = logging.getLogger(__name__) @@ -755,7 +755,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, ---------- query : str SQL-Like Query to return data values - project_id : str (optional when using credentials / service account) + project_id : str (optional when available in environment) Google BigQuery Account project ID. index_col : str (optional) Name of result column to use for index in results DataFrame @@ -892,7 +892,7 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, DataFrame to be written destination_table : string Name of table to be written, in the form 'dataset.tablename' - project_id : str (optional when using credentials / service account) + project_id : str (optional when available in environment) Google BigQuery Account project ID. chunksize : int (default None) Number of rows to be inserted in each chunk from the dataframe. Use From 56436ffd5353bc053cfd4df4aa582c543f2fc792 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 21:54:57 -0400 Subject: [PATCH 13/22] correct mark expression --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b497f803..7dd1ef1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ install: - python setup.py install script: - - if [[ $AUTH == 's_path' ]]; then pytest -m 'not local_auth' -m 'not s_cred_auth' -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ; fi - - if [[ $AUTH == 's_cred' ]]; then pytest -m 'not local_auth' -m 'not s_path_auth' -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ; fi + - if [[ $AUTH == 's_path' ]]; then pytest -m 'not local_auth and not s_cred_auth' -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ; fi + - if [[ $AUTH == 's_cred' ]]; then pytest -m 'not local_auth and not s_path_auth' -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ; fi - if [[ $COVERAGE == 'true' ]]; then codecov ; fi - if [[ $LINT == 'true' ]]; then flake8 pandas_gbq -v ; fi From 5347858646709b3d01ed04095c6e5fc7d7ce6940 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 22:27:41 -0400 Subject: [PATCH 14/22] project not required only if default creds available --- pandas_gbq/tests/test_gbq.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index 8f529184..0f2d1a16 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -310,7 +310,8 @@ def test_to_gbq_should_fail_if_invalid_table_name_passed(self): gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") def test_read_gbq_with_no_project_id_given_should_pass(self, credentials): - gbq.read_gbq('SELECT 1', private_key=credentials) + if _check_if_can_get_correct_default_credentials(): + gbq.read_gbq('SELECT 1') def test_that_parse_data_works_properly(self): From cd9b37d638d9416476845b161fe48941dfd76364 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sat, 31 Mar 2018 23:10:49 -0400 Subject: [PATCH 15/22] remove any more branching in travis --- pandas_gbq/tests/test_gbq.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pandas_gbq/tests/test_gbq.py b/pandas_gbq/tests/test_gbq.py index 0f2d1a16..a4654444 100644 --- a/pandas_gbq/tests/test_gbq.py +++ b/pandas_gbq/tests/test_gbq.py @@ -25,11 +25,6 @@ TABLE_ID = 'new_test' -def _skip_local_auth_if_in_travis_env(): - if _in_travis_environment(): - pytest.skip("Cannot run local auth in travis environment") - - def _skip_if_no_private_key_path(): if not _get_private_key_path(): pytest.skip("Cannot run integration tests without a " @@ -242,8 +237,6 @@ class TestGBQConnectorIntegrationWithLocalUserAccountAuth(object): @pytest.fixture(autouse=True) def setup(self, project): - _skip_local_auth_if_in_travis_env() - self.sut = gbq.GbqConnector(project, auth_local_webserver=True) def test_get_application_default_credentials_does_not_throw_error(self): From a7f6c43434bc21e1d2fc7633d3c5cb9c04060a4a Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Mon, 2 Apr 2018 19:45:34 -0400 Subject: [PATCH 16/22] google import inline --- pandas_gbq/gbq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 41a30df2..b49e7e8f 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -7,7 +7,6 @@ from distutils.version import StrictVersion from time import sleep -import google import numpy as np from pandas import DataFrame, compat from pandas.compat import lzip @@ -172,7 +171,8 @@ def __init__(self, project_id, reauth=False, from google.api_core.exceptions import ClientError self.http_error = (ClientError, GoogleAPIError) if not project_id: - _, project_id = google.auth.default() + from google.auth import default + _, project_id = default() self.project_id = project_id self.reauth = reauth self.private_key = private_key From 7932c59a71132c7eca7677a748dd354d18ed1ddf Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 25 Apr 2018 11:01:49 -0700 Subject: [PATCH 17/22] Use tuple for credentials & project for default project detection. --- pandas_gbq/gbq.py | 33 +++++++++++++++-------- tests/system.py | 60 +++++++++++++----------------------------- tests/unit/test_gbq.py | 44 ++++++++++++++++++++----------- 3 files changed, 70 insertions(+), 67 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index b116300d..38ed51a5 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -189,7 +189,15 @@ def __init__(self, project_id, reauth=False, self.auth_local_webserver = auth_local_webserver self.dialect = dialect self.credentials_path = _get_credentials_file() - self.credentials = self.get_credentials() + self.credentials, default_project = self.get_credentials() + + if self.project_id is None: + self.project_id = default_project + + if self.project_id is None: + raise ValueError( + 'Could not determine project ID and one was not supplied.') + self.client = self.get_client() # BQ Queries costs $5 per TB. First 1 TB per month is free @@ -199,12 +207,13 @@ def __init__(self, project_id, reauth=False, def get_credentials(self): if self.private_key: return self.get_service_account_credentials() - else: - # Try to retrieve Application Default Credentials - credentials = self.get_application_default_credentials() - if not credentials: - credentials = self.get_user_account_credentials() - return credentials + + # Try to retrieve Application Default Credentials + credentials, default_project = self.get_application_default_credentials() + if credentials: + return credentials, default_project + + return self.get_user_account_credentials(), None def get_application_default_credentials(self): """ @@ -230,11 +239,13 @@ def get_application_default_credentials(self): from google.auth.exceptions import DefaultCredentialsError try: - credentials, _ = google.auth.default(scopes=[self.scope]) + credentials, default_project = google.auth.default( + scopes=[self.scope]) except (DefaultCredentialsError, IOError): - return None + return None, None - return _try_credentials(self.project_id, credentials) + billing_project = self.project_id or default_project + return _try_credentials(billing_project, credentials), default_project def load_user_account_credentials(self): """ @@ -415,7 +426,7 @@ def get_service_account_credentials(self): request = google.auth.transport.requests.Request() credentials.refresh(request) - return credentials + return credentials, json_key.get('project_id') except (KeyError, ValueError, TypeError, AttributeError): raise InvalidPrivateKeyFormat( "Private key is missing or invalid. It should be service " diff --git a/tests/system.py b/tests/system.py index a16ed1f3..32d7a34f 100644 --- a/tests/system.py +++ b/tests/system.py @@ -22,6 +22,9 @@ TABLE_ID = 'new_test' +def _skip_local_auth_if_in_travis_env(): + if _in_travis_environment(): + pytest.skip("Cannot run local auth in travis environment") def _skip_if_no_private_key_path(): if not _get_private_key_path(): @@ -167,43 +170,13 @@ def test_generate_bq_schema_deprecated(): gbq.generate_bq_schema(df) -@pytest.fixture(params=[ - pytest.param('local', marks=pytest.mark.local_auth), - pytest.param('service_path', marks=pytest.mark.s_path_auth), - pytest.param('service_creds', marks=pytest.mark.s_cred_auth), -]) -def auth_type(request): - - auth = request.param - - if auth == 'local': - pass - elif auth == 'service_path': - _skip_if_no_private_key_path() - elif auth == 'service_creds': - _skip_if_no_private_key_contents() - else: - raise ValueError - return auth - - @pytest.fixture() -def credentials(auth_type): - - if auth_type == 'local': - return None - - elif auth_type == 'service_path': - return _get_private_key_path() - elif auth_type == 'service_creds': - return _get_private_key_contents() - else: - raise ValueError +def credentials(): + return _get_private_key_contents() @pytest.fixture() def gbq_connector(project, credentials): - return gbq.GbqConnector(project, private_key=credentials) @@ -213,7 +186,7 @@ def test_should_be_able_to_make_a_connector(self, gbq_connector): assert gbq_connector is not None, 'Could not create a GbqConnector' def test_should_be_able_to_get_valid_credentials(self, gbq_connector): - credentials = gbq_connector.get_credentials() + credentials, _ = gbq_connector.get_credentials() assert credentials.valid def test_should_be_able_to_get_a_bigquery_client(self, gbq_connector): @@ -229,13 +202,12 @@ def test_should_be_able_to_get_results_from_query(self, gbq_connector): assert pages is not None -@pytest.mark.local_auth -class TestGBQConnectorIntegrationWithLocalUserAccountAuth(object): +class TestAuth(object): @pytest.fixture(autouse=True) - def setup(self, project): - - self.sut = gbq.GbqConnector(project, auth_local_webserver=True) + def setup(self, gbq_connector): + self.sut = gbq_connector + self.sut.auth_local_webserver = True def test_get_application_default_credentials_does_not_throw_error(self): if _check_if_can_get_correct_default_credentials(): @@ -244,9 +216,9 @@ def test_get_application_default_credentials_does_not_throw_error(self): from google.auth.exceptions import DefaultCredentialsError with mock.patch('google.auth.default', side_effect=DefaultCredentialsError()): - credentials = self.sut.get_application_default_credentials() + credentials, _ = self.sut.get_application_default_credentials() else: - credentials = self.sut.get_application_default_credentials() + credentials, _ = self.sut.get_application_default_credentials() assert credentials is None def test_get_application_default_credentials_returns_credentials(self): @@ -254,10 +226,14 @@ def test_get_application_default_credentials_returns_credentials(self): pytest.skip("Cannot get default_credentials " "from the environment!") from google.auth.credentials import Credentials - credentials = self.sut.get_application_default_credentials() + credentials, default_project = ( + self.sut.get_application_default_credentials()) + assert isinstance(credentials, Credentials) + assert default_project is not None def test_get_user_account_credentials_bad_file_returns_credentials(self): + _skip_local_auth_if_in_travis_env() from google.auth.credentials import Credentials with mock.patch('__main__.open', side_effect=IOError()): @@ -265,6 +241,8 @@ def test_get_user_account_credentials_bad_file_returns_credentials(self): assert isinstance(credentials, Credentials) def test_get_user_account_credentials_returns_credentials(self): + _skip_local_auth_if_in_travis_env() + from google.auth.credentials import Credentials credentials = self.sut.get_user_account_credentials() assert isinstance(credentials, Credentials) diff --git a/tests/unit/test_gbq.py b/tests/unit/test_gbq.py index ae1d35c1..85e4f427 100644 --- a/tests/unit/test_gbq.py +++ b/tests/unit/test_gbq.py @@ -15,9 +15,13 @@ @pytest.fixture(autouse=True) def mock_bigquery_client(monkeypatch): + from google.api_core.exceptions import NotFound import google.cloud.bigquery import google.cloud.bigquery.table mock_client = mock.create_autospec(google.cloud.bigquery.Client) + mock_schema = [ + google.cloud.bigquery.SchemaField('_f0', 'INTEGER') + ] # Mock out SELECT 1 query results. mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) mock_query.job_id = 'some-random-id' @@ -25,11 +29,12 @@ def mock_bigquery_client(monkeypatch): mock_rows = mock.create_autospec( google.cloud.bigquery.table.RowIterator) mock_rows.total_rows = 1 - mock_rows.schema = [ - google.cloud.bigquery.SchemaField('_f0', 'INTEGER')] + mock_rows.schema = mock_schema mock_rows.__iter__.return_value = [(1,)] mock_query.result.return_value = mock_rows mock_client.query.return_value = mock_query + # Mock table creation. + mock_client.get_table.side_effect = NotFound('nope') monkeypatch.setattr( gbq.GbqConnector, 'get_client', lambda _: mock_client) @@ -42,11 +47,7 @@ def no_auth(monkeypatch): monkeypatch.setattr( gbq.GbqConnector, 'get_application_default_credentials', - lambda _: mock_credentials) - monkeypatch.setattr( - gbq.GbqConnector, - 'get_user_account_credentials', - lambda _: mock_credentials) + lambda _: (mock_credentials, 'default-project')) def test_should_return_credentials_path_set_by_env_var(): @@ -76,12 +77,16 @@ def test_should_return_bigquery_correctly_typed( def test_to_gbq_should_fail_if_invalid_table_name_passed(): with pytest.raises(gbq.NotFoundException): - gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") + gbq.to_gbq(DataFrame([[1]]), 'invalid_table_name', project_id="1234") -def test_to_gbq_with_no_project_id_given_should_fail(): +def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): + monkeypatch.setattr( + gbq.GbqConnector, + 'get_application_default_credentials', + lambda _: None) with pytest.raises(TypeError): - gbq.to_gbq(DataFrame(), 'dataset.tablename') + gbq.to_gbq(DataFrame([[1]]), 'dataset.tablename') def test_to_gbq_with_verbose_new_pandas_warns_deprecation(): @@ -95,7 +100,7 @@ def test_to_gbq_with_verbose_new_pandas_warns_deprecation(): mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame(), + DataFrame([[1]]), 'dataset.tablename', project_id='my-project', verbose=True) @@ -114,7 +119,7 @@ def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(): mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame(), + DataFrame([[1]]), 'dataset.tablename', project_id='my-project', verbose=False) @@ -132,7 +137,7 @@ def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn): mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame(), 'dataset.tablename', project_id='my-project') + DataFrame([[1]]), 'dataset.tablename', project_id='my-project') except gbq.TableCreationError: pass assert len(recwarn) == 0 @@ -148,7 +153,7 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn): mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame(), + DataFrame([[1]]), 'dataset.tablename', project_id='my-project', verbose=True) @@ -157,11 +162,20 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn): assert len(recwarn) == 0 -def test_read_gbq_with_no_project_id_given_should_fail(): +def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): + monkeypatch.setattr( + gbq.GbqConnector, + 'get_application_default_credentials', + lambda _: None) with pytest.raises(TypeError): gbq.read_gbq('SELECT 1') +def test_read_gbq_with_inferred_project_id(monkeypatch): + df = gbq.read_gbq('SELECT 1') + assert df is not None + + def test_that_parse_data_works_properly(): from google.cloud.bigquery.table import Row test_schema = {'fields': [ From 31e001fb284815b373d6930a052caa5dc271a6e7 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 25 Apr 2018 11:05:21 -0700 Subject: [PATCH 18/22] lint errors. --- pandas_gbq/gbq.py | 3 ++- tests/system.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 38ed51a5..148dab58 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -209,7 +209,8 @@ def get_credentials(self): return self.get_service_account_credentials() # Try to retrieve Application Default Credentials - credentials, default_project = self.get_application_default_credentials() + credentials, default_project = ( + self.get_application_default_credentials()) if credentials: return credentials, default_project diff --git a/tests/system.py b/tests/system.py index 32d7a34f..c3553c89 100644 --- a/tests/system.py +++ b/tests/system.py @@ -22,10 +22,12 @@ TABLE_ID = 'new_test' + def _skip_local_auth_if_in_travis_env(): if _in_travis_environment(): pytest.skip("Cannot run local auth in travis environment") + def _skip_if_no_private_key_path(): if not _get_private_key_path(): pytest.skip("Cannot run integration tests without a " From 08477ccfddde7fc9ed6449a6fd9eb19b85a8774f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 25 Apr 2018 11:10:32 -0700 Subject: [PATCH 19/22] Remove extra project detection. --- pandas_gbq/gbq.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index 148dab58..b7447074 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -180,9 +180,6 @@ def __init__(self, project_id, reauth=False, from google.api_core.exceptions import GoogleAPIError from google.api_core.exceptions import ClientError self.http_error = (ClientError, GoogleAPIError) - if not project_id: - from google.auth import default - _, project_id = default() self.project_id = project_id self.reauth = reauth self.private_key = private_key From 1f1f2c40e9415d34cbfbce28acce23ba204671d2 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 25 Apr 2018 11:14:44 -0700 Subject: [PATCH 20/22] Update bad_project_id test to query actual data. I think BigQuery stopped checking for valid project on queries with no data access. --- tests/system.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system.py b/tests/system.py index c3553c89..a5115c20 100644 --- a/tests/system.py +++ b/tests/system.py @@ -487,7 +487,8 @@ def test_malformed_query(self): def test_bad_project_id(self): with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq("SELECT 1", project_id='001', + gbq.read_gbq('SELCET * FROM [publicdata:samples.shakespeare]', + project_id='not-my-project', private_key=self.credentials) def test_bad_table_name(self): From d920959f67a03ac279234bb0888a8636f9ea758f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 25 Apr 2018 12:21:34 -0700 Subject: [PATCH 21/22] Skip credentials tests if key not present. --- tests/system.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system.py b/tests/system.py index a5115c20..6f57df3e 100644 --- a/tests/system.py +++ b/tests/system.py @@ -174,6 +174,7 @@ def test_generate_bq_schema_deprecated(): @pytest.fixture() def credentials(): + _skip_if_no_private_key_contents() return _get_private_key_contents() From a3e6d2f2b583d72acb59aeac007631f5837c8d36 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 25 Apr 2018 12:38:11 -0700 Subject: [PATCH 22/22] DOC: add project_id optional to changelog --- docs/source/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index a454ec84..5d7d4dd7 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -4,6 +4,13 @@ Changelog 0.5.0 / TBD ----------- +- Project ID parameter is optional in ``read_gbq`` and ``to_gbq`` when it can + inferred from the environment. Note: you must still pass in a project ID when + using user-based authentication. (:issue:`103`) + +Internal changes +~~~~~~~~~~~~~~~~ + - Tests now use `nox` to run in multiple Python environments. (:issue:`52`) - Renamed internal modules. (:issue:`154`)