Merge pull request #11110 from parthea/gbq-python3

COMPAT: Add Google BigQuery support for python 3 #11094

Author: jreback

doc/source/whatsnew/v0.17.0.txt

@@ -332,6 +332,7 @@ Google BigQuery Enhancements
 - Added ability to replace an existing table and schema when calling the :func:`pandas.io.gbq.to_gbq` function via the ``if_exists`` argument. See the :ref:`docs <io.bigquery>` for more details (:issue:`8325`).
 - ``InvalidColumnOrder`` and ``InvalidPageToken`` in the gbq module will raise ``ValueError`` instead of ``IOError``.
 - The ``generate_bq_schema()`` function is now deprecated and will be removed in a future version (:issue:`11121`)
+- Update the gbq module to support Python 3 (:issue:`11094`).
 
 .. _whatsnew_0170.enhancements.other:
 

pandas/io/gbq.py

@@ -13,22 +13,26 @@
 from pandas.tools.merge import concat
 from pandas.core.common import PandasError
 from pandas.util.decorators import deprecate
+from pandas.compat import lzip, bytes_to_str
 
 def _check_google_client_version():
-    if compat.PY3:
-        raise NotImplementedError("Google's libraries do not support Python 3 yet")
 
     try:
         import pkg_resources
 
     except ImportError:
         raise ImportError('Could not import pkg_resources (setuptools).')
 
+    if compat.PY3:
+        google_api_minimum_version = '1.4.1'
+    else:
+        google_api_minimum_version = '1.2.0'
+
     _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version
 
-    if StrictVersion(_GOOGLE_API_CLIENT_VERSION) < StrictVersion('1.2.0'):
-        raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google "
-                          "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION)
+    if StrictVersion(_GOOGLE_API_CLIENT_VERSION) < StrictVersion(google_api_minimum_version):
+        raise ImportError("pandas requires google-api-python-client >= {0} for Google BigQuery support, "
+                          "current version {1}".format(google_api_minimum_version, _GOOGLE_API_CLIENT_VERSION))
 
 logger = logging.getLogger('pandas.io.gbq')
 logger.setLevel(logging.ERROR)
@@ -161,7 +165,7 @@ def get_service(credentials):
     def process_http_error(ex):
         # See `BigQuery Troubleshooting Errors <https://cloud.google.com/bigquery/troubleshooting-errors>`__
 
-        status = json.loads(ex.content)['error']
+        status = json.loads(bytes_to_str(ex.content))['error']
         errors = status.get('errors', None)
 
         if errors:
@@ -353,10 +357,10 @@ def _parse_data(schema, rows):
 
     fields = schema['fields']
     col_types = [field['type'] for field in fields]
-    col_names = [field['name'].encode('ascii', 'ignore') for field in fields]
+    col_names = [str(field['name']) for field in fields]
     col_dtypes = [dtype_map.get(field['type'], object) for field in fields]
     page_array = np.zeros((len(rows),),
-                          dtype=zip(col_names, col_dtypes))
+                          dtype=lzip(col_names, col_dtypes))
 
     for row_num, raw_row in enumerate(rows):
         entries = raw_row.get('f', [])

pandas/io/tests/test_gbq.py

@@ -30,45 +30,44 @@
 
 
 def _test_imports():
-    if not compat.PY3:
+    global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \
+           _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED
 
-        global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \
-               _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED
-
-        try:
-            import pkg_resources
-            _SETUPTOOLS_INSTALLED = True
-        except ImportError:
-            _SETUPTOOLS_INSTALLED = False
-
-        if _SETUPTOOLS_INSTALLED:
-            try:
-                from apiclient.discovery import build
-                from apiclient.errors import HttpError
+    try:
+        import pkg_resources
+        _SETUPTOOLS_INSTALLED = True
+    except ImportError:
+        _SETUPTOOLS_INSTALLED = False
 
-                from oauth2client.client import OAuth2WebServerFlow
-                from oauth2client.client import AccessTokenRefreshError
+    if compat.PY3:
+        google_api_minimum_version = '1.4.1'
+    else:
+        google_api_minimum_version = '1.2.0'
 
-                from oauth2client.file import Storage
-                from oauth2client.tools import run_flow
-                _GOOGLE_API_CLIENT_INSTALLED=True
-                _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version
+    if _SETUPTOOLS_INSTALLED:
+        try:
+            from apiclient.discovery import build
+            from apiclient.errors import HttpError
 
-                if StrictVersion(_GOOGLE_API_CLIENT_VERSION) >= StrictVersion('1.2.0'):
-                    _GOOGLE_API_CLIENT_VALID_VERSION = True
+            from oauth2client.client import OAuth2WebServerFlow
+            from oauth2client.client import AccessTokenRefreshError
 
-            except ImportError:
-                _GOOGLE_API_CLIENT_INSTALLED = False
+            from oauth2client.file import Storage
+            from oauth2client.tools import run_flow
+            _GOOGLE_API_CLIENT_INSTALLED=True
+            _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version
 
+            if StrictVersion(_GOOGLE_API_CLIENT_VERSION) >= StrictVersion(google_api_minimum_version):
+                _GOOGLE_API_CLIENT_VALID_VERSION = True
 
-            try:
-                import httplib2
-                _HTTPLIB2_INSTALLED = True
-            except ImportError:
-                _HTTPLIB2_INSTALLED = False
+        except ImportError:
+            _GOOGLE_API_CLIENT_INSTALLED = False
 
-    if compat.PY3:
-        raise NotImplementedError("Google's libraries do not support Python 3 yet")
+        try:
+            import httplib2
+            _HTTPLIB2_INSTALLED = True
+        except ImportError:
+            _HTTPLIB2_INSTALLED = False
 
     if not _SETUPTOOLS_INSTALLED:
         raise ImportError('Could not import pkg_resources (setuptools).')
@@ -77,8 +76,8 @@ def _test_imports():
         raise ImportError('Could not import Google API Client.')
 
     if not _GOOGLE_API_CLIENT_VALID_VERSION:
-        raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google "
-                          "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION)
+        raise ImportError("pandas requires google-api-python-client >= {0} for Google BigQuery support, "
+                          "current version {1}".format(google_api_minimum_version, _GOOGLE_API_CLIENT_VERSION))
 
     if not _HTTPLIB2_INSTALLED:
         raise ImportError("pandas requires httplib2 for Google BigQuery support")
@@ -299,7 +298,12 @@ def test_unicode_string_conversion_and_normalization(self):
             {'UNICODE_STRING': [u("\xe9\xfc")]}
         )
 
-        query = 'SELECT "\xc3\xa9\xc3\xbc" as UNICODE_STRING'
+        unicode_string = "\xc3\xa9\xc3\xbc"
+
+        if compat.PY3:
+            unicode_string = unicode_string.encode('latin-1').decode('utf8')
+
+        query = 'SELECT "{0}" as UNICODE_STRING'.format(unicode_string)
 
         df = gbq.read_gbq(query, project_id=PROJECT_ID)
         tm.assert_frame_equal(df, correct_test_datatype)