From 9ec2a68cf421cf8519253252acfb61840af08b0a Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Sun, 10 Jul 2016 22:31:29 +0200 Subject: [PATCH 01/18] Issue #13577 --- doc/source/whatsnew/v0.19.0.txt | 2 ++ pandas/io/gbq.py | 36 ++++++++++++++++++++++++++++++--- pandas/io/tests/test_gbq.py | 7 +++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.19.0.txt b/doc/source/whatsnew/v0.19.0.txt index 657de7ec26efc..2f48b5ac44ada 100644 --- a/doc/source/whatsnew/v0.19.0.txt +++ b/doc/source/whatsnew/v0.19.0.txt @@ -185,6 +185,8 @@ Using the anchoring suffix, you can also specify the day of month to use instead Other enhancements ^^^^^^^^^^^^^^^^^^ +- The ``.get_credentials()`` method of ``GbqConnector`` can now first try to fetch the default credentials for Google Compute Engine without the need to run ``OAuth2WebServerFlow`` - if private_key is not provided (:issue:`13577`) + - The ``.tz_localize()`` method of ``DatetimeIndex`` and ``Timestamp`` has gained the ``errors`` keyword, so you can potentially coerce nonexistent timestamps to ``NaT``. The default behaviour remains to raising a ``NonExistentTimeError`` (:issue:`13057`) - ``Index`` now supports ``.str.extractall()`` which returns a ``DataFrame``, see :ref:`documentation here ` (:issue:`10008`, :issue:`13156`) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 140f5cc6bb6e3..c26d8431ffe57 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -159,7 +159,33 @@ def get_credentials(self): if self.private_key: return self.get_service_account_credentials() else: - return self.get_user_account_credentials() + # Try to retrieve Application Default Credentials + credentials = self.get_application_default_credentials() + if not credentials: + credentials = self.get_user_account_credentials() + return credentials + + def get_application_default_credentials(self): + from oauth2client.client import GoogleCredentials + from oauth2client.client import AccessTokenRefreshError + from oauth2client.client import ApplicationDefaultCredentialsError + from apiclient.discovery import build + from apiclient.errors import HttpError + + credentials = None + try: + credentials = GoogleCredentials.get_application_default() + except ApplicationDefaultCredentialsError: + return None + # Check if the application has rights to the BigQuery project + bigquery_service = build('bigquery', 'v2', credentials=credentials) + job_collection = bigquery_service.jobs() + job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} + try: + job_collection.insert(projectId=self.project_id, body=job_data).execute() + except (AccessTokenRefreshError, HttpError): + return None + return credentials def get_user_account_credentials(self): from oauth2client.client import OAuth2WebServerFlow @@ -576,7 +602,9 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, https://developers.google.com/api-client-library/python/apis/bigquery/v2 Authentication to the Google BigQuery service is via OAuth 2.0. - By default user account credentials are used. You will be asked to + By default "application default credentials" are used. + If default application credentials are not found or are restrictive - + User account credentials are used. You will be asked to grant permissions for product name 'pandas GBQ'. It is also posible to authenticate via service account credentials by using private_key parameter. @@ -672,7 +700,9 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, https://developers.google.com/api-client-library/python/apis/bigquery/v2 Authentication to the Google BigQuery service is via OAuth 2.0. - By default user account credentials are used. You will be asked to + By default "application default credentials" are used. + If default application credentials are not found or are restrictive - + User account credentials are used. You will be asked to grant permissions for product name 'pandas GBQ'. It is also posible to authenticate via service account credentials by using private_key parameter. diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 278c5d7215624..250a95552863b 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -80,8 +80,10 @@ def _test_imports(): from apiclient.discovery import build # noqa from apiclient.errors import HttpError # noqa + from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.client import AccessTokenRefreshError # noqa + from oauth2client.client import ApplicationDefaultCredentialsError # noqa from oauth2client.file import Storage # noqa from oauth2client.tools import run_flow # noqa @@ -217,6 +219,11 @@ def test_should_be_able_to_get_results_from_query(self): schema, pages = self.sut.run_query('SELECT 1') self.assertTrue(pages is not None) + def test_should_be_able_to_get_a_bigquery_service_from_default_credentials(self): + from oauth2client.client import GoogleCredentials + credentials = self.sut.get_application_default_credentials() + self.assertTrue(isinstance(credentials, (type(None), GoogleCredentials))) + class TestGBQConnectorServiceAccountKeyPathIntegration(tm.TestCase): def setUp(self): From 17f4740cc5697e9302983a798272ff72a13b8aa7 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Sun, 10 Jul 2016 23:01:56 +0200 Subject: [PATCH 02/18] Issue #13577 - flake8 fixes --- pandas/io/gbq.py | 4 ++-- pandas/io/tests/test_gbq.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index c26d8431ffe57..eae470bd69f42 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -179,10 +179,10 @@ def get_application_default_credentials(self): return None # Check if the application has rights to the BigQuery project bigquery_service = build('bigquery', 'v2', credentials=credentials) - job_collection = bigquery_service.jobs() + jobs = bigquery_service.jobs() job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} try: - job_collection.insert(projectId=self.project_id, body=job_data).execute() + jobs.insert(projectId=self.project_id, body=job_data).execute() except (AccessTokenRefreshError, HttpError): return None return credentials diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 250a95552863b..1ba7bb27a9727 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -83,7 +83,7 @@ def _test_imports(): from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.client import AccessTokenRefreshError # noqa - from oauth2client.client import ApplicationDefaultCredentialsError # noqa + from oauth2client.client import ApplicationDefaultCredentialsError # noqa from oauth2client.file import Storage # noqa from oauth2client.tools import run_flow # noqa @@ -219,10 +219,11 @@ def test_should_be_able_to_get_results_from_query(self): schema, pages = self.sut.run_query('SELECT 1') self.assertTrue(pages is not None) - def test_should_be_able_to_get_a_bigquery_service_from_default_credentials(self): + def test_should_be_able_to_get_credentials_from_default_credentials(self): from oauth2client.client import GoogleCredentials credentials = self.sut.get_application_default_credentials() - self.assertTrue(isinstance(credentials, (type(None), GoogleCredentials))) + valid_types = (type(None), GoogleCredentials) + self.assertTrue(isinstance(credentials, valid_types)) class TestGBQConnectorServiceAccountKeyPathIntegration(tm.TestCase): From 9740938bc8b16a64605b4c28d234bb08bb817e28 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Mon, 11 Jul 2016 14:56:32 +0200 Subject: [PATCH 03/18] concav unable to capture coverage --- pandas/io/gbq.py | 10 ++++++---- pandas/io/tests/test_gbq.py | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index eae470bd69f42..ccad45a0bd6b9 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -160,12 +160,14 @@ def get_credentials(self): return self.get_service_account_credentials() else: # Try to retrieve Application Default Credentials - credentials = self.get_application_default_credentials() + project_id = self.project_id + credentials = self.get_application_default_credentials(project_id) if not credentials: credentials = self.get_user_account_credentials() return credentials - def get_application_default_credentials(self): + @staticmethod + def get_application_default_credentials(project_id): from oauth2client.client import GoogleCredentials from oauth2client.client import AccessTokenRefreshError from oauth2client.client import ApplicationDefaultCredentialsError @@ -182,8 +184,8 @@ def get_application_default_credentials(self): jobs = bigquery_service.jobs() job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} try: - jobs.insert(projectId=self.project_id, body=job_data).execute() - except (AccessTokenRefreshError, HttpError): + jobs.insert(projectId=project_id, body=job_data).execute() + except (AccessTokenRefreshError, HttpError, TypeError): return None return credentials diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 1ba7bb27a9727..63c959f72136f 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -189,6 +189,20 @@ def test_generate_bq_schema_deprecated(): df = make_mixed_dataframe_v2(10) gbq.generate_bq_schema(df) +def google_credentials_import(): + try: + from oauth2client.client import GoogleCredentials + return GoogleCredentials + except ImportError: + return type(None) + +def test_should_be_able_to_get_credentials_from_default_credentials(): + GoogleCredentials = google_credentials_import() + connector = gbq.GbqConnector + credentials = connector.get_application_default_credentials(PROJECT_ID) + valid_types = (type(None), GoogleCredentials) + assert isinstance(credentials, valid_types) + class TestGBQConnectorIntegration(tm.TestCase): @@ -219,12 +233,6 @@ def test_should_be_able_to_get_results_from_query(self): schema, pages = self.sut.run_query('SELECT 1') self.assertTrue(pages is not None) - def test_should_be_able_to_get_credentials_from_default_credentials(self): - from oauth2client.client import GoogleCredentials - credentials = self.sut.get_application_default_credentials() - valid_types = (type(None), GoogleCredentials) - self.assertTrue(isinstance(credentials, valid_types)) - class TestGBQConnectorServiceAccountKeyPathIntegration(tm.TestCase): def setUp(self): From a02b620591bc98c3e5e316395e0dac8f84fc4e5d Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Tue, 12 Jul 2016 11:06:34 +0200 Subject: [PATCH 04/18] unit test fix - if libraries can not be imported --- pandas/io/gbq.py | 24 ++++++++++++++++++------ pandas/io/tests/test_gbq.py | 9 ++++++--- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index ccad45a0bd6b9..ef2d254cf1b11 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -168,13 +168,25 @@ def get_credentials(self): @staticmethod def get_application_default_credentials(project_id): - from oauth2client.client import GoogleCredentials - from oauth2client.client import AccessTokenRefreshError - from oauth2client.client import ApplicationDefaultCredentialsError - from apiclient.discovery import build - from apiclient.errors import HttpError - + """ + Given a project_id tries to retrieve the + "default application credentials" + Could be useful for running code on Google Cloud Platform + """ credentials = None + try: + from oauth2client.client import GoogleCredentials + from oauth2client.client import AccessTokenRefreshError + from oauth2client.client import ApplicationDefaultCredentialsError + try: + from googleapiclient.discovery import build + from googleapiclient.errors import HttpError + except: + from apiclient.discovery import build + from apiclient.errors import HttpError + except ImportError: + return None + try: credentials = GoogleCredentials.get_application_default() except ApplicationDefaultCredentialsError: diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 63c959f72136f..0b0aacf9b9ca7 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -83,7 +83,8 @@ def _test_imports(): from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.client import AccessTokenRefreshError # noqa - from oauth2client.client import ApplicationDefaultCredentialsError # noqa + from oauth2client.client import \ + ApplicationDefaultCredentialsError # noqa from oauth2client.file import Storage # noqa from oauth2client.tools import run_flow # noqa @@ -137,7 +138,8 @@ def _test_imports(): oauth2client_v1 = False try: - from oauth2client.service_account import ServiceAccountCredentials # noqa + from oauth2client.service_account import \ + ServiceAccountCredentials # noqa except ImportError: oauth2client_v2 = False @@ -159,7 +161,8 @@ def clean_gbq_environment(private_key=None): for i in range(1, 10): if DATASET_ID + str(i) in dataset.datasets(): dataset_id = DATASET_ID + str(i) - table = gbq._Table(PROJECT_ID, dataset_id, private_key=private_key) + table = gbq._Table(PROJECT_ID, dataset_id, + private_key=private_key) for j in range(1, 20): if TABLE_ID + str(j) in dataset.tables(dataset_id): table.delete(TABLE_ID + str(j)) From fca8003e04353fb686d685f3b73da55331c15efb Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Tue, 12 Jul 2016 15:11:22 +0200 Subject: [PATCH 05/18] lint errors --- pandas/io/gbq.py | 2 +- pandas/io/tests/test_gbq.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index ef2d254cf1b11..236ce07846a69 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -169,7 +169,7 @@ def get_credentials(self): @staticmethod def get_application_default_credentials(project_id): """ - Given a project_id tries to retrieve the + Given a project_id tries to retrieve the "default application credentials" Could be useful for running code on Google Cloud Platform """ diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 0b0aacf9b9ca7..118723fe8d981 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -83,8 +83,6 @@ def _test_imports(): from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.client import AccessTokenRefreshError # noqa - from oauth2client.client import \ - ApplicationDefaultCredentialsError # noqa from oauth2client.file import Storage # noqa from oauth2client.tools import run_flow # noqa @@ -138,8 +136,7 @@ def _test_imports(): oauth2client_v1 = False try: - from oauth2client.service_account import \ - ServiceAccountCredentials # noqa + from oauth2client.service_account import ServiceAccountCredentials # noqa except ImportError: oauth2client_v2 = False @@ -161,8 +158,7 @@ def clean_gbq_environment(private_key=None): for i in range(1, 10): if DATASET_ID + str(i) in dataset.datasets(): dataset_id = DATASET_ID + str(i) - table = gbq._Table(PROJECT_ID, dataset_id, - private_key=private_key) + table = gbq._Table(PROJECT_ID, dataset_id, private_key=private_key) for j in range(1, 20): if TABLE_ID + str(j) in dataset.tables(dataset_id): table.delete(TABLE_ID + str(j)) @@ -192,6 +188,7 @@ def test_generate_bq_schema_deprecated(): df = make_mixed_dataframe_v2(10) gbq.generate_bq_schema(df) + def google_credentials_import(): try: from oauth2client.client import GoogleCredentials @@ -199,6 +196,7 @@ def google_credentials_import(): except ImportError: return type(None) + def test_should_be_able_to_get_credentials_from_default_credentials(): GoogleCredentials = google_credentials_import() connector = gbq.GbqConnector From 7f1fd2634ce058eadfa1ad321897ee85ea7750b3 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Tue, 12 Jul 2016 22:03:40 +0200 Subject: [PATCH 06/18] Coverage was giving wrong percentage Coverage gives wrong percentage if imports are missing --- pandas/io/gbq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 236ce07846a69..c1691dbc12cc1 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -167,7 +167,7 @@ def get_credentials(self): return credentials @staticmethod - def get_application_default_credentials(project_id): + def get_application_default_credentials(project_id): # pragma: no cover """ Given a project_id tries to retrieve the "default application credentials" From d4c7e3d33eca9d88a308802c8921addec40b77b9 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Thu, 14 Jul 2016 10:31:24 +0200 Subject: [PATCH 07/18] Issue #13577 - Feedback incorporation --- pandas/io/gbq.py | 56 ++++++++++++++++++------------------- pandas/io/tests/test_gbq.py | 22 ++++----------- 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index c1691dbc12cc1..c1459e08f9d13 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -160,33 +160,30 @@ def get_credentials(self): return self.get_service_account_credentials() else: # Try to retrieve Application Default Credentials - project_id = self.project_id - credentials = self.get_application_default_credentials(project_id) + credentials = self.get_application_default_credentials() if not credentials: credentials = self.get_user_account_credentials() return credentials - @staticmethod - def get_application_default_credentials(project_id): # pragma: no cover + def get_application_default_credentials(self): """ - Given a project_id tries to retrieve the - "default application credentials" + This method tries to retrieve the "default application credentials" Could be useful for running code on Google Cloud Platform """ - credentials = None + from oauth2client.client import AccessTokenRefreshError + try: + from googleapiclient.discovery import build + from googleapiclient.errors import HttpError + except: + from apiclient.discovery import build + from apiclient.errors import HttpError try: from oauth2client.client import GoogleCredentials - from oauth2client.client import AccessTokenRefreshError from oauth2client.client import ApplicationDefaultCredentialsError - try: - from googleapiclient.discovery import build - from googleapiclient.errors import HttpError - except: - from apiclient.discovery import build - from apiclient.errors import HttpError except ImportError: return None + credentials = None try: credentials = GoogleCredentials.get_application_default() except ApplicationDefaultCredentialsError: @@ -196,7 +193,7 @@ def get_application_default_credentials(project_id): # pragma: no cover jobs = bigquery_service.jobs() job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} try: - jobs.insert(projectId=project_id, body=job_data).execute() + jobs.insert(projectId=self.project_id, body=job_data).execute() except (AccessTokenRefreshError, HttpError, TypeError): return None return credentials @@ -616,12 +613,14 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, https://developers.google.com/api-client-library/python/apis/bigquery/v2 Authentication to the Google BigQuery service is via OAuth 2.0. - By default "application default credentials" are used. - If default application credentials are not found or are restrictive - - User account credentials are used. You will be asked to - grant permissions for product name 'pandas GBQ'. It is also posible - to authenticate via service account credentials by using - private_key parameter. + If "private_key" is not provided: + By default "application default credentials" are used [new behavior] + If default application credentials are not found or are restrictive - + User account credentials are used. In this case - you will be asked to + grant permissions for product name 'pandas GBQ'. + If "private_key" is provided: + It is also posible to authenticate via service account credentials + by using this parameter. Parameters ---------- @@ -713,13 +712,14 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, Documentation is available at https://developers.google.com/api-client-library/python/apis/bigquery/v2 - Authentication to the Google BigQuery service is via OAuth 2.0. - By default "application default credentials" are used. - If default application credentials are not found or are restrictive - - User account credentials are used. You will be asked to - grant permissions for product name 'pandas GBQ'. It is also posible - to authenticate via service account credentials by using - private_key parameter. + If "private_key" is not provided: + By default "application default credentials" are used [new behavior] + If default application credentials are not found or are restrictive - + User account credentials are used. In this case - you will be asked to + grant permissions for product name 'pandas GBQ'. + If "private_key" is provided: + It is also posible to authenticate via service account credentials + by using this parameter. Parameters ---------- diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 118723fe8d981..e70dcc129f4bf 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -189,22 +189,6 @@ def test_generate_bq_schema_deprecated(): gbq.generate_bq_schema(df) -def google_credentials_import(): - try: - from oauth2client.client import GoogleCredentials - return GoogleCredentials - except ImportError: - return type(None) - - -def test_should_be_able_to_get_credentials_from_default_credentials(): - GoogleCredentials = google_credentials_import() - connector = gbq.GbqConnector - credentials = connector.get_application_default_credentials(PROJECT_ID) - valid_types = (type(None), GoogleCredentials) - assert isinstance(credentials, valid_types) - - class TestGBQConnectorIntegration(tm.TestCase): def setUp(self): @@ -234,6 +218,12 @@ def test_should_be_able_to_get_results_from_query(self): schema, pages = self.sut.run_query('SELECT 1') self.assertTrue(pages is not None) + def test_get_application_default_credentials_should_not_throw_error(self): + from oauth2client.client import GoogleCredentials + credentials = self.sut.get_application_default_credentials() + valid_types = (type(None), GoogleCredentials) + assert isinstance(credentials, valid_types) + class TestGBQConnectorServiceAccountKeyPathIntegration(tm.TestCase): def setUp(self): From 17dd814b6ea3cab0132815e2c3f3bc14eb6e3e0c Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Sun, 7 Aug 2016 19:49:23 +0200 Subject: [PATCH 08/18] splitting integration tests for --- pandas/io/gbq.py | 4 +++- pandas/io/tests/test_gbq.py | 32 +++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index a3246f74f5a6c..becff544847bf 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -180,6 +180,7 @@ def get_application_default_credentials(self): from apiclient.errors import HttpError try: from oauth2client.client import GoogleCredentials + from oauth2client.client import HttpAccessTokenRefreshError from oauth2client.client import ApplicationDefaultCredentialsError except ImportError: return None @@ -195,7 +196,8 @@ def get_application_default_credentials(self): job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} try: jobs.insert(projectId=self.project_id, body=job_data).execute() - except (AccessTokenRefreshError, HttpError, TypeError): + except (HttpAccessTokenRefreshError, AccessTokenRefreshError, + HttpError, TypeError): return None return credentials diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 33f9c04a83d5a..e97767bb72cc2 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -56,6 +56,27 @@ def _skip_if_no_private_key_contents(): _skip_if_no_private_key_contents() +def _skip_if_get_correct_default_credentials(can=False): + got_credentials = False + try: + from oauth2client.client import GoogleCredentials + from apiclient.discovery import build + credentials = GoogleCredentials.get_application_default() + bigquery_service = build('bigquery', 'v2', credentials=credentials) + jobs = bigquery_service.jobs() + job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} + jobs.insert(projectId=PROJECT_ID, body=job_data).execute() + got_credentials = True + except: + pass + if can and got_credentials: + raise nose.SkipTest("Cannot get default_credentials " + "from the environment!") + if (not can) and (not got_credentials): + raise nose.SkipTest("Can get default_credentials " + "from the environment!") + + def _test_imports(): global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \ _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED @@ -218,11 +239,16 @@ def test_should_be_able_to_get_results_from_query(self): schema, pages = self.sut.run_query('SELECT 1') self.assertTrue(pages is not None) - def test_get_application_default_credentials_should_not_throw_error(self): + def test_get_application_default_credentials_does_not_throw_error(self): + _skip_if_get_correct_default_credentials(can=True) + credentials = self.sut.get_application_default_credentials() + self.assertIsNone(credentials) + + def test_get_application_default_credentials_returns_credentials(self): + _skip_if_get_correct_default_credentials(can=False) from oauth2client.client import GoogleCredentials credentials = self.sut.get_application_default_credentials() - valid_types = (type(None), GoogleCredentials) - assert isinstance(credentials, valid_types) + self.assertTrue(isinstance(credentials, GoogleCredentials)) class TestGBQConnectorServiceAccountKeyPathIntegration(tm.TestCase): From 5a2dd159ccd884403f059616a1c9595713c80942 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Wed, 10 Aug 2016 09:47:22 +0100 Subject: [PATCH 09/18] feedback changes --- pandas/io/gbq.py | 50 +++++++++++++++++++++---------------- pandas/io/tests/test_gbq.py | 21 ++++++++++------ 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index becff544847bf..f0c53c1d95f88 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -171,18 +171,20 @@ def get_application_default_credentials(self): This method tries to retrieve the "default application credentials" Could be useful for running code on Google Cloud Platform """ - from oauth2client.client import AccessTokenRefreshError - try: - from googleapiclient.discovery import build - from googleapiclient.errors import HttpError - except: - from apiclient.discovery import build - from apiclient.errors import HttpError + oauth2client_library_imported = False try: from oauth2client.client import GoogleCredentials + from oauth2client.client import AccessTokenRefreshError from oauth2client.client import HttpAccessTokenRefreshError from oauth2client.client import ApplicationDefaultCredentialsError + oauth2client_library_imported = True + from googleapiclient.discovery import build + from googleapiclient.errors import HttpError except ImportError: + from apiclient.discovery import build + from apiclient.errors import HttpError + + if not oauth2client_library_imported: return None credentials = None @@ -190,6 +192,7 @@ def get_application_default_credentials(self): credentials = GoogleCredentials.get_application_default() except ApplicationDefaultCredentialsError: return None + # Check if the application has rights to the BigQuery project bigquery_service = build('bigquery', 'v2', credentials=credentials) jobs = bigquery_service.jobs() @@ -617,14 +620,16 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, https://developers.google.com/api-client-library/python/apis/bigquery/v2 Authentication to the Google BigQuery service is via OAuth 2.0. - If "private_key" is not provided: - By default "application default credentials" are used [new behavior] - If default application credentials are not found or are restrictive - - User account credentials are used. In this case - you will be asked to + - If "private_key" is not provided: + By default "application default credentials" are used. + + .. versionadded:: 0.19.0 + + If default application credentials are not found or are restrictive, + user account credentials are used. In this case, you will be asked to grant permissions for product name 'pandas GBQ'. - If "private_key" is provided: - It is also posible to authenticate via service account credentials - by using this parameter. + - If "private_key" is provided: + Service account credentials will be used to authenticate. Parameters ---------- @@ -731,14 +736,17 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, Documentation is available at https://developers.google.com/api-client-library/python/apis/bigquery/v2 - If "private_key" is not provided: - By default "application default credentials" are used [new behavior] - If default application credentials are not found or are restrictive - - User account credentials are used. In this case - you will be asked to + Authentication to the Google BigQuery service is via OAuth 2.0. + - If "private_key" is not provided: + By default "application default credentials" are used. + + .. versionadded:: 0.19.0 + + If default application credentials are not found or are restrictive, + user account credentials are used. In this case, you will be asked to grant permissions for product name 'pandas GBQ'. - If "private_key" is provided: - It is also posible to authenticate via service account credentials - by using this parameter. + - If "private_key" is provided: + Service account credentials will be used to authenticate. Parameters ---------- diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index e97767bb72cc2..079a642e453b2 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -56,8 +56,7 @@ def _skip_if_no_private_key_contents(): _skip_if_no_private_key_contents() -def _skip_if_get_correct_default_credentials(can=False): - got_credentials = False +def _check_if_can_get_correct_default_credentials(): try: from oauth2client.client import GoogleCredentials from apiclient.discovery import build @@ -66,13 +65,19 @@ def _skip_if_get_correct_default_credentials(can=False): jobs = bigquery_service.jobs() job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} jobs.insert(projectId=PROJECT_ID, body=job_data).execute() - got_credentials = True + return True except: - pass - if can and got_credentials: + return False + + +def _skip_if_cant_get_correct_default_credentials(): + if not _check_if_can_get_correct_default_credentials(): raise nose.SkipTest("Cannot get default_credentials " "from the environment!") - if (not can) and (not got_credentials): + + +def _skip_if_can_get_correct_default_credentials(): + if _check_if_can_get_correct_default_credentials(): raise nose.SkipTest("Can get default_credentials " "from the environment!") @@ -240,12 +245,12 @@ def test_should_be_able_to_get_results_from_query(self): self.assertTrue(pages is not None) def test_get_application_default_credentials_does_not_throw_error(self): - _skip_if_get_correct_default_credentials(can=True) + _skip_if_can_get_correct_default_credentials() credentials = self.sut.get_application_default_credentials() self.assertIsNone(credentials) def test_get_application_default_credentials_returns_credentials(self): - _skip_if_get_correct_default_credentials(can=False) + _skip_if_cant_get_correct_default_credentials() from oauth2client.client import GoogleCredentials credentials = self.sut.get_application_default_credentials() self.assertTrue(isinstance(credentials, GoogleCredentials)) From 64134acb3e455faf69dace5ca93559b432d5ee4f Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Wed, 10 Aug 2016 09:54:28 +0100 Subject: [PATCH 10/18] apiclient import fix --- pandas/io/gbq.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index f0c53c1d95f88..7ddd15b4d7563 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -181,12 +181,11 @@ def get_application_default_credentials(self): from googleapiclient.discovery import build from googleapiclient.errors import HttpError except ImportError: + if not oauth2client_library_imported: + return None from apiclient.discovery import build from apiclient.errors import HttpError - if not oauth2client_library_imported: - return None - credentials = None try: credentials = GoogleCredentials.get_application_default() From 5fd1775095669b524ba56286f32fe9ab44a40295 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Wed, 10 Aug 2016 12:16:37 +0100 Subject: [PATCH 11/18] added more documentation --- doc/source/whatsnew/v0.19.0.txt | 2 +- pandas/io/gbq.py | 39 +++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/doc/source/whatsnew/v0.19.0.txt b/doc/source/whatsnew/v0.19.0.txt index f6c425d900fa9..a14a80168af2b 100644 --- a/doc/source/whatsnew/v0.19.0.txt +++ b/doc/source/whatsnew/v0.19.0.txt @@ -361,7 +361,7 @@ Google BigQuery Enhancements Other enhancements ^^^^^^^^^^^^^^^^^^ -- The ``.get_credentials()`` method of ``GbqConnector`` can now first try to fetch the default credentials for Google Compute Engine without the need to run ``OAuth2WebServerFlow`` - if private_key is not provided (:issue:`13577`) +- The ``.get_credentials()`` method of ``GbqConnector`` can now first try to fetch the application default credentials (https://developers.google.com/identity/protocols/application-default-credentials). This is only valid if the parameter ``private_key`` is not provided. Using this functionality, the ``OAuth2WebServerFlow`` call can be skipped (:issue:`13577`). - The ``.tz_localize()`` method of ``DatetimeIndex`` and ``Timestamp`` has gained the ``errors`` keyword, so you can potentially coerce nonexistent timestamps to ``NaT``. The default behaviour remains to raising a ``NonExistentTimeError`` (:issue:`13057`) - ``pd.to_numeric()`` now accepts a ``downcast`` parameter, which will downcast the data if possible to smallest specified numerical dtype (:issue:`13352`) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 7ddd15b4d7563..905a7ed6f909c 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -52,7 +52,10 @@ def _test_google_api_imports(): except: from apiclient.discovery import build # noqa from apiclient.errors import HttpError # noqa + from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import AccessTokenRefreshError # noqa + from oauth2client.client import HttpAccessTokenRefreshError # noqa + from oauth2client.client import ApplicationDefaultCredentialsError # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.file import Storage # noqa from oauth2client.tools import run_flow, argparser # noqa @@ -168,25 +171,37 @@ def get_credentials(self): def get_application_default_credentials(self): """ - This method tries to retrieve the "default application credentials" - Could be useful for running code on Google Cloud Platform + This method tries to retrieve the "default application credentials". + This could be useful for running code on Google Cloud Platform. + + .. versionadded:: 0.19.0 + + Parameters + ---------- + None + + Returns + ------- + - GoogleCredentials, + If the default application credentials can be retrieved + from the environment. The retrieved credentials should also + have access to the project (self.project_id) on BigQuery. + - OR None, + If default application credentials can not be retrieved + from the environment. Or, the retrieved credentials do not + have access to the project (self.project_id) on BigQuery. """ - oauth2client_library_imported = False try: - from oauth2client.client import GoogleCredentials - from oauth2client.client import AccessTokenRefreshError - from oauth2client.client import HttpAccessTokenRefreshError - from oauth2client.client import ApplicationDefaultCredentialsError - oauth2client_library_imported = True from googleapiclient.discovery import build from googleapiclient.errors import HttpError except ImportError: - if not oauth2client_library_imported: - return None from apiclient.discovery import build from apiclient.errors import HttpError + from oauth2client.client import GoogleCredentials + from oauth2client.client import AccessTokenRefreshError + from oauth2client.client import HttpAccessTokenRefreshError + from oauth2client.client import ApplicationDefaultCredentialsError - credentials = None try: credentials = GoogleCredentials.get_application_default() except ApplicationDefaultCredentialsError: @@ -198,10 +213,10 @@ def get_application_default_credentials(self): job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} try: jobs.insert(projectId=self.project_id, body=job_data).execute() + return credentials except (HttpAccessTokenRefreshError, AccessTokenRefreshError, HttpError, TypeError): return None - return credentials def get_user_account_credentials(self): from oauth2client.client import OAuth2WebServerFlow From 62815e1ad71f9dd56e902666fc55ec001c871713 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Wed, 10 Aug 2016 20:06:39 +0200 Subject: [PATCH 12/18] Import errors --- pandas/io/gbq.py | 12 ++------ pandas/io/tests/test_gbq.py | 56 ++++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 905a7ed6f909c..70c7c57e70438 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -54,8 +54,6 @@ def _test_google_api_imports(): from apiclient.errors import HttpError # noqa from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import AccessTokenRefreshError # noqa - from oauth2client.client import HttpAccessTokenRefreshError # noqa - from oauth2client.client import ApplicationDefaultCredentialsError # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.file import Storage # noqa from oauth2client.tools import run_flow, argparser # noqa @@ -193,18 +191,13 @@ def get_application_default_credentials(self): """ try: from googleapiclient.discovery import build - from googleapiclient.errors import HttpError except ImportError: from apiclient.discovery import build - from apiclient.errors import HttpError from oauth2client.client import GoogleCredentials - from oauth2client.client import AccessTokenRefreshError - from oauth2client.client import HttpAccessTokenRefreshError - from oauth2client.client import ApplicationDefaultCredentialsError try: credentials = GoogleCredentials.get_application_default() - except ApplicationDefaultCredentialsError: + except: return None # Check if the application has rights to the BigQuery project @@ -214,8 +207,7 @@ def get_application_default_credentials(self): try: jobs.insert(projectId=self.project_id, body=job_data).execute() return credentials - except (HttpAccessTokenRefreshError, AccessTokenRefreshError, - HttpError, TypeError): + except: return None def get_user_account_credentials(self): diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 079a642e453b2..3cd8d24b04221 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -56,32 +56,6 @@ def _skip_if_no_private_key_contents(): _skip_if_no_private_key_contents() -def _check_if_can_get_correct_default_credentials(): - try: - from oauth2client.client import GoogleCredentials - from apiclient.discovery import build - credentials = GoogleCredentials.get_application_default() - bigquery_service = build('bigquery', 'v2', credentials=credentials) - jobs = bigquery_service.jobs() - job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} - jobs.insert(projectId=PROJECT_ID, body=job_data).execute() - return True - except: - return False - - -def _skip_if_cant_get_correct_default_credentials(): - if not _check_if_can_get_correct_default_credentials(): - raise nose.SkipTest("Cannot get default_credentials " - "from the environment!") - - -def _skip_if_can_get_correct_default_credentials(): - if _check_if_can_get_correct_default_credentials(): - raise nose.SkipTest("Can get default_credentials " - "from the environment!") - - def _test_imports(): global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \ _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED @@ -178,6 +152,36 @@ def test_requirements(): raise nose.SkipTest(import_exception) +def _check_if_can_get_correct_default_credentials(): + test_requirements() + from oauth2client.client import GoogleCredentials + try: + from googleapiclient.discovery import build + except ImportError: + from apiclient.discovery import build + try: + credentials = GoogleCredentials.get_application_default() + bigquery_service = build('bigquery', 'v2', credentials=credentials) + jobs = bigquery_service.jobs() + job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} + jobs.insert(projectId=PROJECT_ID, body=job_data).execute() + return True + except: + return False + + +def _skip_if_cant_get_correct_default_credentials(): + if not _check_if_can_get_correct_default_credentials(): + raise nose.SkipTest("Cannot get default_credentials " + "from the environment!") + + +def _skip_if_can_get_correct_default_credentials(): + if _check_if_can_get_correct_default_credentials(): + raise nose.SkipTest("Can get default_credentials " + "from the environment!") + + def clean_gbq_environment(private_key=None): dataset = gbq._Dataset(PROJECT_ID, private_key=private_key) From cf41e760f917ccfaa08a4906e499b16635dc179f Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Thu, 11 Aug 2016 09:23:21 +0100 Subject: [PATCH 13/18] modified documentation + restructured tests --- doc/source/io.rst | 9 +++++++++ doc/source/whatsnew/v0.19.0.txt | 2 +- pandas/io/tests/test_gbq.py | 23 +++++++++-------------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/doc/source/io.rst b/doc/source/io.rst index 7917e6b4cdfce..a7d891b927015 100644 --- a/doc/source/io.rst +++ b/doc/source/io.rst @@ -4475,6 +4475,15 @@ Additional information on service accounts can be found You will need to install an additional dependency: `oauth2client `__. +.. versionadded:: 0.19.0 + +Authentication via ``application default credentials`` is also possible. This is only valid +if the parameter ``private_key`` is not provided. This method also requires that +the credentials can be fetched from the environment the code is running in. +Otherwise, the OAuth2 client-side authentication is used. +Additional information on ``application default credentials`` can be found +`here `__. + .. note:: The `'private_key'` parameter can be set to either the file path of the service account key diff --git a/doc/source/whatsnew/v0.19.0.txt b/doc/source/whatsnew/v0.19.0.txt index a14a80168af2b..8b4543bff4076 100644 --- a/doc/source/whatsnew/v0.19.0.txt +++ b/doc/source/whatsnew/v0.19.0.txt @@ -361,7 +361,7 @@ Google BigQuery Enhancements Other enhancements ^^^^^^^^^^^^^^^^^^ -- The ``.get_credentials()`` method of ``GbqConnector`` can now first try to fetch the application default credentials (https://developers.google.com/identity/protocols/application-default-credentials). This is only valid if the parameter ``private_key`` is not provided. Using this functionality, the ``OAuth2WebServerFlow`` call can be skipped (:issue:`13577`). +- The ``.get_credentials()`` method of ``GbqConnector`` can now first try to fetch [the application default credentials](https://developers.google.com/identity/protocols/application-default-credentials). This is only valid if the parameter ``private_key`` is not provided. Using this functionality, the ``OAuth2WebServerFlow`` call can be skipped (:issue:`13577`). - The ``.tz_localize()`` method of ``DatetimeIndex`` and ``Timestamp`` has gained the ``errors`` keyword, so you can potentially coerce nonexistent timestamps to ``NaT``. The default behaviour remains to raising a ``NonExistentTimeError`` (:issue:`13057`) - ``pd.to_numeric()`` now accepts a ``downcast`` parameter, which will downcast the data if possible to smallest specified numerical dtype (:issue:`13352`) diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 3cd8d24b04221..8d3581f5f33f0 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -153,6 +153,9 @@ def test_requirements(): def _check_if_can_get_correct_default_credentials(): + # Checks if "Application Default Credentials" can be fetched + # from the environment the tests are running in. + # See Issue #13577 test_requirements() from oauth2client.client import GoogleCredentials try: @@ -170,18 +173,6 @@ def _check_if_can_get_correct_default_credentials(): return False -def _skip_if_cant_get_correct_default_credentials(): - if not _check_if_can_get_correct_default_credentials(): - raise nose.SkipTest("Cannot get default_credentials " - "from the environment!") - - -def _skip_if_can_get_correct_default_credentials(): - if _check_if_can_get_correct_default_credentials(): - raise nose.SkipTest("Can get default_credentials " - "from the environment!") - - def clean_gbq_environment(private_key=None): dataset = gbq._Dataset(PROJECT_ID, private_key=private_key) @@ -249,12 +240,16 @@ def test_should_be_able_to_get_results_from_query(self): self.assertTrue(pages is not None) def test_get_application_default_credentials_does_not_throw_error(self): - _skip_if_can_get_correct_default_credentials() + if _check_if_can_get_correct_default_credentials(): + raise nose.SkipTest("Can get default_credentials " + "from the environment!") credentials = self.sut.get_application_default_credentials() self.assertIsNone(credentials) def test_get_application_default_credentials_returns_credentials(self): - _skip_if_cant_get_correct_default_credentials() + if not _check_if_can_get_correct_default_credentials(): + raise nose.SkipTest("Cannot get default_credentials " + "from the environment!") from oauth2client.client import GoogleCredentials credentials = self.sut.get_application_default_credentials() self.assertTrue(isinstance(credentials, GoogleCredentials)) From ac48104208e80893db26f1569f8a0ee4f57858d3 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Thu, 11 Aug 2016 21:25:00 +0200 Subject: [PATCH 14/18] Updated whatsnew entry --- doc/source/whatsnew/v0.19.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.19.0.txt b/doc/source/whatsnew/v0.19.0.txt index 8b4543bff4076..fa1b21116cb4f 100644 --- a/doc/source/whatsnew/v0.19.0.txt +++ b/doc/source/whatsnew/v0.19.0.txt @@ -361,7 +361,7 @@ Google BigQuery Enhancements Other enhancements ^^^^^^^^^^^^^^^^^^ -- The ``.get_credentials()`` method of ``GbqConnector`` can now first try to fetch [the application default credentials](https://developers.google.com/identity/protocols/application-default-credentials). This is only valid if the parameter ``private_key`` is not provided. Using this functionality, the ``OAuth2WebServerFlow`` call can be skipped (:issue:`13577`). +- The ``.get_credentials()`` method of ``GbqConnector`` can now first try to fetch [the application default credentials](https://developers.google.com/identity/protocols/application-default-credentials). See the :ref:`docs ` for more details (:issue:`13577`). - The ``.tz_localize()`` method of ``DatetimeIndex`` and ``Timestamp`` has gained the ``errors`` keyword, so you can potentially coerce nonexistent timestamps to ``NaT``. The default behaviour remains to raising a ``NonExistentTimeError`` (:issue:`13057`) - ``pd.to_numeric()`` now accepts a ``downcast`` parameter, which will downcast the data if possible to smallest specified numerical dtype (:issue:`13352`) From 4cf26aa77d1d0e444965ff23ed8329ebdbad31d3 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Sun, 14 Aug 2016 17:10:19 +0200 Subject: [PATCH 15/18] handling GoogleCredentials import error --- pandas/io/gbq.py | 6 ++++-- pandas/io/tests/test_gbq.py | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 70c7c57e70438..b2f095a296a05 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -52,7 +52,6 @@ def _test_google_api_imports(): except: from apiclient.discovery import build # noqa from apiclient.errors import HttpError # noqa - from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import AccessTokenRefreshError # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.file import Storage # noqa @@ -193,7 +192,10 @@ def get_application_default_credentials(self): from googleapiclient.discovery import build except ImportError: from apiclient.discovery import build - from oauth2client.client import GoogleCredentials + try: + from oauth2client.client import GoogleCredentials + except ImportError: + return None try: credentials = GoogleCredentials.get_application_default() diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 8d3581f5f33f0..4d7d7ccd6783d 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -80,7 +80,6 @@ def _test_imports(): from apiclient.discovery import build # noqa from apiclient.errors import HttpError # noqa - from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.client import AccessTokenRefreshError # noqa @@ -157,12 +156,12 @@ def _check_if_can_get_correct_default_credentials(): # from the environment the tests are running in. # See Issue #13577 test_requirements() - from oauth2client.client import GoogleCredentials try: from googleapiclient.discovery import build except ImportError: from apiclient.discovery import build try: + from oauth2client.client import GoogleCredentials credentials = GoogleCredentials.get_application_default() bigquery_service = build('bigquery', 'v2', credentials=credentials) jobs = bigquery_service.jobs() From 6fe392a826e87ccf789ac5495cd327e183f109af Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Sun, 14 Aug 2016 21:56:59 +0200 Subject: [PATCH 16/18] API error on google-api-python-client==1.2 --- pandas/io/gbq.py | 11 +++++++---- pandas/io/tests/test_gbq.py | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index b2f095a296a05..326e32c84ebe6 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -188,6 +188,7 @@ def get_application_default_credentials(self): from the environment. Or, the retrieved credentials do not have access to the project (self.project_id) on BigQuery. """ + import httplib2 try: from googleapiclient.discovery import build except ImportError: @@ -202,11 +203,13 @@ def get_application_default_credentials(self): except: return None - # Check if the application has rights to the BigQuery project - bigquery_service = build('bigquery', 'v2', credentials=credentials) - jobs = bigquery_service.jobs() - job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} + http = httplib2.Http() try: + http = credentials.authorize(http) + bigquery_service = build('bigquery', 'v2', http=http) + # Check if the application has rights to the BigQuery project + jobs = bigquery_service.jobs() + job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} jobs.insert(projectId=self.project_id, body=job_data).execute() return credentials except: diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 4d7d7ccd6783d..4b71192c907f8 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -156,6 +156,7 @@ def _check_if_can_get_correct_default_credentials(): # from the environment the tests are running in. # See Issue #13577 test_requirements() + import httplib2 try: from googleapiclient.discovery import build except ImportError: @@ -163,7 +164,9 @@ def _check_if_can_get_correct_default_credentials(): try: from oauth2client.client import GoogleCredentials credentials = GoogleCredentials.get_application_default() - bigquery_service = build('bigquery', 'v2', credentials=credentials) + http = httplib2.Http() + http = credentials.authorize(http) + bigquery_service = build('bigquery', 'v2', http=http) jobs = bigquery_service.jobs() job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} jobs.insert(projectId=PROJECT_ID, body=job_data).execute() From 6bbdcdc0a85ab08e8414051034e1409fb3086209 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Wed, 17 Aug 2016 12:19:30 +0100 Subject: [PATCH 17/18] fixed documentation --- doc/source/io.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/source/io.rst b/doc/source/io.rst index a7d891b927015..b0ea88268c10f 100644 --- a/doc/source/io.rst +++ b/doc/source/io.rst @@ -4475,14 +4475,13 @@ Additional information on service accounts can be found You will need to install an additional dependency: `oauth2client `__. -.. versionadded:: 0.19.0 - Authentication via ``application default credentials`` is also possible. This is only valid if the parameter ``private_key`` is not provided. This method also requires that the credentials can be fetched from the environment the code is running in. Otherwise, the OAuth2 client-side authentication is used. -Additional information on ``application default credentials`` can be found -`here `__. +Additional information on `application default credentials `__. + +.. versionadded:: 0.19.0 .. note:: From a65b3f170582343fcb7fdede77f1c4136702e161 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Wed, 17 Aug 2016 14:49:36 +0100 Subject: [PATCH 18/18] newline in documentation --- doc/source/io.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/io.rst b/doc/source/io.rst index b0ea88268c10f..477e1a7d5026a 100644 --- a/doc/source/io.rst +++ b/doc/source/io.rst @@ -4479,7 +4479,8 @@ Authentication via ``application default credentials`` is also possible. This is if the parameter ``private_key`` is not provided. This method also requires that the credentials can be fetched from the environment the code is running in. Otherwise, the OAuth2 client-side authentication is used. -Additional information on `application default credentials `__. +Additional information on +`application default credentials `__. .. versionadded:: 0.19.0