diff --git a/doc/source/whatsnew/v0.16.1.txt b/doc/source/whatsnew/v0.16.1.txt index 84472802d73be..db9d970e1aba1 100755 --- a/doc/source/whatsnew/v0.16.1.txt +++ b/doc/source/whatsnew/v0.16.1.txt @@ -254,3 +254,5 @@ Bug Fixes - Bug in hiding ticklabels with subplots and shared axes when adding a new plot to an existing grid of axes (:issue:`9158`) - Bug in ``transform`` and ``filter`` when grouping on a categorical variable (:issue:`9921`) - Bug in ``transform`` when groups are equal in number and dtype to the input index (:issue:`9700`) +- Google BigQuery connector now imports dependencies on a per-method basis.(:issue:`9713`) +- Updated BigQuery connector to no longer use deprecated ```oauth2client.tools.run()``` (:issue:`8327`) diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 9ecffb382e151..f1fcc822adeaf 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -13,75 +13,22 @@ from pandas.tools.merge import concat from pandas.core.common import PandasError -_IMPORTS = False -_GOOGLE_API_CLIENT_INSTALLED = False -_GOOGLE_API_CLIENT_VALID_VERSION = False -_GOOGLE_FLAGS_INSTALLED = False -_GOOGLE_FLAGS_VALID_VERSION = False -_HTTPLIB2_INSTALLED = False -_SETUPTOOLS_INSTALLED = False -def _importers(): - # import things we need - # but make this done on a first use basis - - global _IMPORTS - if _IMPORTS: - return - - _IMPORTS = True - - if not compat.PY3: - - global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \ - _GOOGLE_FLAGS_INSTALLED, _GOOGLE_FLAGS_VALID_VERSION, \ - _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED - - try: - import pkg_resources - _SETUPTOOLS_INSTALLED = True - except ImportError: - _SETUPTOOLS_INSTALLED = False - - if _SETUPTOOLS_INSTALLED: - try: - from apiclient.discovery import build - from apiclient.http import MediaFileUpload - from apiclient.errors import HttpError - - from oauth2client.client import OAuth2WebServerFlow - from oauth2client.client import AccessTokenRefreshError - from oauth2client.client import flow_from_clientsecrets - from oauth2client.file import Storage - from oauth2client.tools import run - _GOOGLE_API_CLIENT_INSTALLED=True - _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version - - if LooseVersion(_GOOGLE_API_CLIENT_VERSION) >= '1.2.0': - _GOOGLE_API_CLIENT_VALID_VERSION = True - - except ImportError: - _GOOGLE_API_CLIENT_INSTALLED = False - - - try: - import gflags as flags - _GOOGLE_FLAGS_INSTALLED = True - - _GOOGLE_FLAGS_VERSION = pkg_resources.get_distribution('python-gflags').version +def _check_google_client_version(): + if compat.PY3: + raise NotImplementedError("Google's libraries do not support Python 3 yet") - if LooseVersion(_GOOGLE_FLAGS_VERSION) >= '2.0': - _GOOGLE_FLAGS_VALID_VERSION = True + try: + import pkg_resources - except ImportError: - _GOOGLE_FLAGS_INSTALLED = False + except ImportError: + raise ImportError('Could not import pkg_resources (setuptools).') - try: - import httplib2 - _HTTPLIB2_INSTALLED = True - except ImportError: - _HTTPLIB2_INSTALLED = False + _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version + if LooseVersion(_GOOGLE_API_CLIENT_VERSION) < '1.2.0': + raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google " + "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION) logger = logging.getLogger('pandas.io.gbq') logger.setLevel(logging.ERROR) @@ -142,6 +89,16 @@ def __init__(self, project_id, reauth=False): self.service = self.get_service(self.credentials) def get_credentials(self): + try: + from oauth2client.client import OAuth2WebServerFlow + from oauth2client.file import Storage + from oauth2client.tools import run_flow, argparser + + except ImportError: + raise ImportError('Could not import Google API Client.') + + _check_google_client_version() + flow = OAuth2WebServerFlow(client_id='495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd.apps.googleusercontent.com', client_secret='kOc9wMptUtxkcIFbtZCcrEAc', scope='https://www.googleapis.com/auth/bigquery', @@ -151,11 +108,25 @@ def get_credentials(self): credentials = storage.get() if credentials is None or credentials.invalid or self.reauth: - credentials = run(flow, storage) + credentials = run_flow(flow, storage, argparser.parse_args([])) return credentials def get_service(self, credentials): + try: + import httplib2 + + except ImportError: + raise ImportError("pandas requires httplib2 for Google BigQuery support") + + try: + from apiclient.discovery import build + + except ImportError: + raise ImportError('Could not import Google API Client.') + + _check_google_client_version() + http = httplib2.Http() http = credentials.authorize(http) bigquery_service = build('bigquery', 'v2', http=http) @@ -163,6 +134,15 @@ def get_service(self, credentials): return bigquery_service def run_query(self, query): + try: + from apiclient.errors import HttpError + from oauth2client.client import AccessTokenRefreshError + + except ImportError: + raise ImportError('Could not import Google API Client.') + + _check_google_client_version() + job_collection = self.service.jobs() job_data = { 'configuration': { @@ -313,38 +293,6 @@ def _parse_entry(field_value, field_type): return field_value == 'true' return field_value -def _test_imports(): - - _importers() - _GOOGLE_API_CLIENT_INSTALLED - _GOOGLE_API_CLIENT_VALID_VERSION - _GOOGLE_FLAGS_INSTALLED - _GOOGLE_FLAGS_VALID_VERSION - _HTTPLIB2_INSTALLED - _SETUPTOOLS_INSTALLED - - if compat.PY3: - raise NotImplementedError("Google's libraries do not support Python 3 yet") - - if not _SETUPTOOLS_INSTALLED: - raise ImportError('Could not import pkg_resources (setuptools).') - - if not _GOOGLE_API_CLIENT_INSTALLED: - raise ImportError('Could not import Google API Client.') - - if not _GOOGLE_FLAGS_INSTALLED: - raise ImportError('Could not import Google Command Line Flags Module.') - - if not _GOOGLE_API_CLIENT_VALID_VERSION: - raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google " - "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION) - - if not _GOOGLE_FLAGS_VALID_VERSION: - raise ImportError("pandas requires python-gflags >= 2.0.0 for Google " - "BigQuery support, current version " + _GOOGLE_FLAGS_VERSION) - - if not _HTTPLIB2_INSTALLED: - raise ImportError("pandas requires httplib2 for Google BigQuery support") def read_gbq(query, project_id = None, index_col=None, col_order=None, reauth=False): """Load data from Google BigQuery. @@ -379,7 +327,6 @@ def read_gbq(query, project_id = None, index_col=None, col_order=None, reauth=Fa """ - _test_imports() if not project_id: raise TypeError("Missing required parameter: project_id") @@ -450,7 +397,6 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=10000, if multiple accounts are used. """ - _test_imports() if not project_id: raise TypeError("Missing required parameter: project_id") diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index 2f79cc8ba1826..5417842d3f863 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -12,6 +12,9 @@ import numpy as np +from distutils.version import LooseVersion +from pandas import compat + from pandas import NaT from pandas.compat import u from pandas.core.frame import DataFrame @@ -22,6 +25,12 @@ VERSION = platform.python_version() +_IMPORTS = False +_GOOGLE_API_CLIENT_INSTALLED = False +_GOOGLE_API_CLIENT_VALID_VERSION = False +_HTTPLIB2_INSTALLED = False +_SETUPTOOLS_INSTALLED = False + def missing_bq(): try: subprocess.call('bq') @@ -29,9 +38,64 @@ def missing_bq(): except OSError: return True +def _test_imports(): + if not compat.PY3: + + global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \ + _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED + + try: + import pkg_resources + _SETUPTOOLS_INSTALLED = True + except ImportError: + _SETUPTOOLS_INSTALLED = False + + if _SETUPTOOLS_INSTALLED: + try: + from apiclient.discovery import build + from apiclient.errors import HttpError + + from oauth2client.client import OAuth2WebServerFlow + from oauth2client.client import AccessTokenRefreshError + + from oauth2client.file import Storage + from oauth2client.tools import run_flow + _GOOGLE_API_CLIENT_INSTALLED=True + _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version + + if LooseVersion(_GOOGLE_API_CLIENT_VERSION) >= '1.2.0': + _GOOGLE_API_CLIENT_VALID_VERSION = True + + except ImportError: + _GOOGLE_API_CLIENT_INSTALLED = False + + + try: + import httplib2 + _HTTPLIB2_INSTALLED = True + except ImportError: + _HTTPLIB2_INSTALLED = False + + + if compat.PY3: + raise NotImplementedError("Google's libraries do not support Python 3 yet") + + if not _SETUPTOOLS_INSTALLED: + raise ImportError('Could not import pkg_resources (setuptools).') + + if not _GOOGLE_API_CLIENT_INSTALLED: + raise ImportError('Could not import Google API Client.') + + if not _GOOGLE_API_CLIENT_VALID_VERSION: + raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google " + "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION) + + if not _HTTPLIB2_INSTALLED: + raise ImportError("pandas requires httplib2 for Google BigQuery support") + def test_requirements(): try: - gbq._test_imports() + _test_imports() except (ImportError, NotImplementedError) as import_exception: raise nose.SkipTest(import_exception)