Resolve ImportError in read_gbq which appears with oauthclient >= 2.0

closes #12572
closes #12582

Author: tonypartheniou

ci/requirements-2.7.pip

@@ -2,5 +2,6 @@ blosc
 httplib2
 google-api-python-client == 1.2
 python-gflags == 2.0
+oauth2client == 1.5.0
 pathlib
 py

ci/requirements-3.4.pip

@@ -2,3 +2,4 @@ python-dateutil==2.2
 blosc
 httplib2
 google-api-python-client
+oauth2client

doc/source/install.rst

@@ -267,9 +267,11 @@ Optional Dependencies
   <http://www.vergenet.net/~conrad/software/xsel/>`__, or `xclip
   <http://sourceforge.net/projects/xclip/>`__: necessary to use
   :func:`~pandas.io.clipboard.read_clipboard`. Most package managers on Linux distributions will have ``xclip`` and/or ``xsel`` immediately available for installation.
-* Google's `python-gflags <http://code.google.com/p/python-gflags/>`__
-  and `google-api-python-client <http://github.com/google/google-api-python-client>`__: Needed for :mod:`~pandas.io.gbq`
-* `httplib2 <http://pypi.python.org/pypi/httplib2>`__: Needed for :mod:`~pandas.io.gbq`
+* Google's `python-gflags <http://code.google.com/p/python-gflags/>`__ ,
+  `oauth2client <https://github.com/google/oauth2client>`__ ,
+  `httplib2 <http://pypi.python.org/pypi/httplib2>`__
+  and `google-api-python-client <http://github.com/google/google-api-python-client>`__
+  : Needed for :mod:`~pandas.io.gbq`
 * One of the following combinations of libraries is needed to use the
   top-level :func:`~pandas.io.html.read_html` function:
 

doc/source/io.rst

@@ -4168,6 +4168,12 @@ DataFrame with a shape and data types derived from the source table.
 Additionally, DataFrames can be inserted into new BigQuery tables or appended
 to existing tables.
 
+You will need to install some additional dependencies:
+
+- Google's `python-gflags <http://code.google.com/p/python-gflags/>`__
+- `httplib2 <http://pypi.python.org/pypi/httplib2>`__
+- `google-api-python-client <http://github.com/google/google-api-python-client>`__
+
 .. warning::
 
    To use this module, you will need a valid BigQuery account. Refer to the
@@ -4206,6 +4212,8 @@ is particularly useful when working on remote servers (eg. jupyter iPython noteb
 Additional information on service accounts can be found
 `here <https://developers.google.com/identity/protocols/OAuth2#serviceaccount>`__.
 
+You will need to install an additional dependency: `oauth2client <https://github.com/google/oauth2client>`__.
+
 .. note::
 
    The `'private_key'` parameter can be set to either the file path of the service account key

doc/source/whatsnew/v0.18.0.txt

@@ -511,7 +511,7 @@ Other enhancements
 - Added ``DataFrame.style.format`` for more flexible formatting of cell values (:issue:`11692`)
 - ``DataFrame.select_dtypes`` now allows the ``np.float16`` typecode (:issue:`11990`)
 - ``pivot_table()`` now accepts most iterables for the ``values`` parameter (:issue:`12017`)
-- Added Google ``BigQuery`` service account authentication support, which enables authentication on remote servers. (:issue:`11881`). For further details see :ref:`here <io.bigquery_authentication>`
+- Added Google ``BigQuery`` service account authentication support, which enables authentication on remote servers. (:issue:`11881`, :issue:`12572`). For further details see :ref:`here <io.bigquery_authentication>`
 - ``HDFStore`` is now iterable: ``for k in store`` is equivalent to ``for k in store.keys()`` (:issue:`12221`).
 - Add missing methods/fields to ``.dt`` for ``Period`` (:issue:`8848`)
 - The entire codebase has been ``PEP``-ified (:issue:`12096`)

pandas/io/gbq.py

@@ -50,7 +50,6 @@ def _test_google_api_imports():
         from apiclient.errors import HttpError  # noqa
         from oauth2client.client import AccessTokenRefreshError  # noqa
         from oauth2client.client import OAuth2WebServerFlow  # noqa
-        from oauth2client.client import SignedJwtAssertionCredentials  # noqa
         from oauth2client.file import Storage  # noqa
         from oauth2client.tools import run_flow, argparser  # noqa
     except ImportError as e:
@@ -179,7 +178,30 @@ def get_user_account_credentials(self):
         return credentials
 
     def get_service_account_credentials(self):
-        from oauth2client.client import SignedJwtAssertionCredentials
+        # Bug fix for https://github.com/pydata/pandas/issues/12572
+        # We need to know that a supported version of oauth2client is installed
+        # Test that either of the following is installed:
+        # - SignedJwtAssertionCredentials from oauth2client.client
+        # - ServiceAccountCredentials from oauth2client.service_account
+        # SignedJwtAssertionCredentials is available in oauthclient < 2.0.0
+        # ServiceAccountCredentials is available in oauthclient >= 2.0.0
+        oauth2client_v1 = True
+        oauth2client_v2 = True
+
+        try:
+            from oauth2client.client import SignedJwtAssertionCredentials
+        except ImportError:
+            oauth2client_v1 = False
+
+        try:
+            from oauth2client.service_account import ServiceAccountCredentials
+        except ImportError:
+            oauth2client_v2 = False
+
+        if not oauth2client_v1 and not oauth2client_v2:
+            raise ImportError("Missing oauth2client required for BigQuery "
+                              "service account support")
+
         from os.path import isfile
 
         try:
@@ -197,11 +219,16 @@ def get_service_account_credentials(self):
                 json_key['private_key'] = bytes(
                     json_key['private_key'], 'UTF-8')
 
-            return SignedJwtAssertionCredentials(
-                json_key['client_email'],
-                json_key['private_key'],
-                self.scope,
-            )
+            if oauth2client_v1:
+                return SignedJwtAssertionCredentials(
+                    json_key['client_email'],
+                    json_key['private_key'],
+                    self.scope,
+                )
+            else:
+                return ServiceAccountCredentials.from_json_keyfile_dict(
+                    json_key,
+                    self.scope)
         except (KeyError, ValueError, TypeError, AttributeError):
             raise InvalidPrivateKeyFormat(
                 "Private key is missing or invalid. It should be service "

pandas/io/tests/test_gbq.py

@@ -77,7 +77,6 @@ def _test_imports():
 
             from oauth2client.client import OAuth2WebServerFlow  # noqa
             from oauth2client.client import AccessTokenRefreshError  # noqa
-            from oauth2client.client import SignedJwtAssertionCredentials  # noqa
 
             from oauth2client.file import Storage  # noqa
             from oauth2client.tools import run_flow  # noqa
@@ -115,6 +114,30 @@ def _test_imports():
         raise ImportError(
             "pandas requires httplib2 for Google BigQuery support")
 
+    # Bug fix for https://github.com/pydata/pandas/issues/12572
+    # We need to know that a supported version of oauth2client is installed
+    # Test that either of the following is installed:
+    # - SignedJwtAssertionCredentials from oauth2client.client
+    # - ServiceAccountCredentials from oauth2client.service_account
+    # SignedJwtAssertionCredentials is available in oauthclient < 2.0.0
+    # ServiceAccountCredentials is available in oauthclient >= 2.0.0
+    oauth2client_v1 = True
+    oauth2client_v2 = True
+
+    try:
+        from oauth2client.client import SignedJwtAssertionCredentials  # noqa
+    except ImportError:
+        oauth2client_v1 = False
+
+    try:
+        from oauth2client.service_account import ServiceAccountCredentials  # noqa
+    except ImportError:
+        oauth2client_v2 = False
+
+    if not oauth2client_v1 and not oauth2client_v2:
+        raise ImportError("Missing oauth2client required for BigQuery "
+                          "service account support")
+
 
 def test_requirements():
     try: