From 41ec5af5a18a357dbe05cda873a2d63f08a7b645 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Thu, 7 Jul 2016 10:32:33 +0200 Subject: [PATCH 1/5] Default credentials for Google Compute Engine --- pandas/io/gbq.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 140f5cc6bb6e3..91325fe8c8beb 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -165,6 +165,12 @@ def get_user_account_credentials(self): from oauth2client.client import OAuth2WebServerFlow from oauth2client.file import Storage from oauth2client.tools import run_flow, argparser + from oauth2client.client import GoogleCredentials, ApplicationDefaultCredentialsError + + try: + return GoogleCredentials.get_application_default() + except ApplicationDefaultCredentialsError: + pass flow = OAuth2WebServerFlow( client_id=('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd' From fd898e3edd3cd515d1799cec0ad9e18d1569c338 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Thu, 7 Jul 2016 11:18:40 +0200 Subject: [PATCH 2/5] Change for Issue #13577 --- doc/source/whatsnew/v0.18.2.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.18.2.txt b/doc/source/whatsnew/v0.18.2.txt index 64644bd9a7a26..8495c09ca380b 100644 --- a/doc/source/whatsnew/v0.18.2.txt +++ b/doc/source/whatsnew/v0.18.2.txt @@ -234,6 +234,7 @@ API changes - An ``UnsupportedFunctionCall`` error is now raised if NumPy ufuncs like ``np.mean`` are called on groupby or resample objects (:issue:`12811`) - Calls to ``.sample()`` will respect the random seed set via ``numpy.random.seed(n)`` (:issue:`13161`) - ``Styler.apply`` is now more strict about the outputs your function must return. For ``axis=0`` or ``axis=1``, the output shape must be identical. For ``axis=None``, the output must be a DataFrame with identical columns and index labels. (:issue:`13222`) +- ``io\gbq.py`` `GbqConnector` can now fetch the default credentials on Google Compute Engine without the need to run `OAuth2WebServerFlow` (:issue:`13577`) .. _whatsnew_0182.api.tolist: From 5f492105c9dcc8e6aad937fd207cc855f9209033 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Fri, 8 Jul 2016 11:09:21 +0200 Subject: [PATCH 3/5] Checking if default application credentials have rights to the BigQuery project --- pandas/io/gbq.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 91325fe8c8beb..c26d8431ffe57 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -159,18 +159,38 @@ def get_credentials(self): if self.private_key: return self.get_service_account_credentials() else: - return self.get_user_account_credentials() + # Try to retrieve Application Default Credentials + credentials = self.get_application_default_credentials() + if not credentials: + credentials = self.get_user_account_credentials() + return credentials + + def get_application_default_credentials(self): + from oauth2client.client import GoogleCredentials + from oauth2client.client import AccessTokenRefreshError + from oauth2client.client import ApplicationDefaultCredentialsError + from apiclient.discovery import build + from apiclient.errors import HttpError + + credentials = None + try: + credentials = GoogleCredentials.get_application_default() + except ApplicationDefaultCredentialsError: + return None + # Check if the application has rights to the BigQuery project + bigquery_service = build('bigquery', 'v2', credentials=credentials) + job_collection = bigquery_service.jobs() + job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} + try: + job_collection.insert(projectId=self.project_id, body=job_data).execute() + except (AccessTokenRefreshError, HttpError): + return None + return credentials def get_user_account_credentials(self): from oauth2client.client import OAuth2WebServerFlow from oauth2client.file import Storage from oauth2client.tools import run_flow, argparser - from oauth2client.client import GoogleCredentials, ApplicationDefaultCredentialsError - - try: - return GoogleCredentials.get_application_default() - except ApplicationDefaultCredentialsError: - pass flow = OAuth2WebServerFlow( client_id=('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd' @@ -582,7 +602,9 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, https://developers.google.com/api-client-library/python/apis/bigquery/v2 Authentication to the Google BigQuery service is via OAuth 2.0. - By default user account credentials are used. You will be asked to + By default "application default credentials" are used. + If default application credentials are not found or are restrictive - + User account credentials are used. You will be asked to grant permissions for product name 'pandas GBQ'. It is also posible to authenticate via service account credentials by using private_key parameter. @@ -678,7 +700,9 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, https://developers.google.com/api-client-library/python/apis/bigquery/v2 Authentication to the Google BigQuery service is via OAuth 2.0. - By default user account credentials are used. You will be asked to + By default "application default credentials" are used. + If default application credentials are not found or are restrictive - + User account credentials are used. You will be asked to grant permissions for product name 'pandas GBQ'. It is also posible to authenticate via service account credentials by using private_key parameter. From b9cc14a7ed6512d7f9edeb1b23360fb5b72185ad Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Fri, 8 Jul 2016 11:21:01 +0200 Subject: [PATCH 4/5] added imports for test_requirements --- pandas/io/tests/test_gbq.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 278c5d7215624..0cbf682480678 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -80,8 +80,10 @@ def _test_imports(): from apiclient.discovery import build # noqa from apiclient.errors import HttpError # noqa + from oauth2client.client import GoogleCredentials # noqa from oauth2client.client import OAuth2WebServerFlow # noqa from oauth2client.client import AccessTokenRefreshError # noqa + from oauth2client.client import ApplicationDefaultCredentialsError # noqa from oauth2client.file import Storage # noqa from oauth2client.tools import run_flow # noqa From 58c9a981666d80bfe573366a4e625086dd41bb49 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb Tariq Date: Fri, 8 Jul 2016 11:32:28 +0200 Subject: [PATCH 5/5] Moved the change from API changes to Enhancements --- doc/source/whatsnew/v0.18.2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.18.2.txt b/doc/source/whatsnew/v0.18.2.txt index 8495c09ca380b..ac5fceb672809 100644 --- a/doc/source/whatsnew/v0.18.2.txt +++ b/doc/source/whatsnew/v0.18.2.txt @@ -223,6 +223,7 @@ Other enhancements - A ``union_categorical`` function has been added for combining categoricals, see :ref:`Unioning Categoricals` (:issue:`13361`) - ``eval``'s upcasting rules for ``float32`` types have been updated to be more consistent with NumPy's rules. New behavior will not upcast to ``float64`` if you multiply a pandas ``float32`` object by a scalar float64. (:issue:`12388`) - ``Series`` has gained the properties ``.is_monotonic``, ``.is_monotonic_increasing``, ``.is_monotonic_decreasing``, similar to ``Index`` (:issue:`13336`) +- The ``.get_credentials()`` method of ``GbqConnector`` can now first try to fetch the default credentials for Google Compute Engine without the need to run ``OAuth2WebServerFlow`` - if private_key is not provided (:issue:`13577`) .. _whatsnew_0182.api: @@ -234,7 +235,6 @@ API changes - An ``UnsupportedFunctionCall`` error is now raised if NumPy ufuncs like ``np.mean`` are called on groupby or resample objects (:issue:`12811`) - Calls to ``.sample()`` will respect the random seed set via ``numpy.random.seed(n)`` (:issue:`13161`) - ``Styler.apply`` is now more strict about the outputs your function must return. For ``axis=0`` or ``axis=1``, the output shape must be identical. For ``axis=None``, the output must be a DataFrame with identical columns and index labels. (:issue:`13222`) -- ``io\gbq.py`` `GbqConnector` can now fetch the default credentials on Google Compute Engine without the need to run `OAuth2WebServerFlow` (:issue:`13577`) .. _whatsnew_0182.api.tolist: