diff --git a/doc/source/whatsnew/v0.17.0.txt b/doc/source/whatsnew/v0.17.0.txt index 03cac12436898..c8ab9599cd92a 100644 --- a/doc/source/whatsnew/v0.17.0.txt +++ b/doc/source/whatsnew/v0.17.0.txt @@ -332,6 +332,7 @@ Google BigQuery Enhancements - Added ability to replace an existing table and schema when calling the :func:`pandas.io.gbq.to_gbq` function via the ``if_exists`` argument. See the :ref:`docs ` for more details (:issue:`8325`). - ``InvalidColumnOrder`` and ``InvalidPageToken`` in the gbq module will raise ``ValueError`` instead of ``IOError``. - The ``generate_bq_schema()`` function is now deprecated and will be removed in a future version (:issue:`11121`) +- Update the gbq module to support Python 3 (:issue:`11094`). .. _whatsnew_0170.enhancements.other: diff --git a/pandas/io/gbq.py b/pandas/io/gbq.py index 22500f23f6e07..e9568db06f391 100644 --- a/pandas/io/gbq.py +++ b/pandas/io/gbq.py @@ -13,10 +13,9 @@ from pandas.tools.merge import concat from pandas.core.common import PandasError from pandas.util.decorators import deprecate +from pandas.compat import lzip, bytes_to_str def _check_google_client_version(): - if compat.PY3: - raise NotImplementedError("Google's libraries do not support Python 3 yet") try: import pkg_resources @@ -24,11 +23,16 @@ def _check_google_client_version(): except ImportError: raise ImportError('Could not import pkg_resources (setuptools).') + if compat.PY3: + google_api_minimum_version = '1.4.1' + else: + google_api_minimum_version = '1.2.0' + _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version - if StrictVersion(_GOOGLE_API_CLIENT_VERSION) < StrictVersion('1.2.0'): - raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google " - "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION) + if StrictVersion(_GOOGLE_API_CLIENT_VERSION) < StrictVersion(google_api_minimum_version): + raise ImportError("pandas requires google-api-python-client >= {0} for Google BigQuery support, " + "current version {1}".format(google_api_minimum_version, _GOOGLE_API_CLIENT_VERSION)) logger = logging.getLogger('pandas.io.gbq') logger.setLevel(logging.ERROR) @@ -161,7 +165,7 @@ def get_service(credentials): def process_http_error(ex): # See `BigQuery Troubleshooting Errors `__ - status = json.loads(ex.content)['error'] + status = json.loads(bytes_to_str(ex.content))['error'] errors = status.get('errors', None) if errors: @@ -353,10 +357,10 @@ def _parse_data(schema, rows): fields = schema['fields'] col_types = [field['type'] for field in fields] - col_names = [field['name'].encode('ascii', 'ignore') for field in fields] + col_names = [str(field['name']) for field in fields] col_dtypes = [dtype_map.get(field['type'], object) for field in fields] page_array = np.zeros((len(rows),), - dtype=zip(col_names, col_dtypes)) + dtype=lzip(col_names, col_dtypes)) for row_num, raw_row in enumerate(rows): entries = raw_row.get('f', []) diff --git a/pandas/io/tests/test_gbq.py b/pandas/io/tests/test_gbq.py index b57530e926a2e..cc1e901d8f119 100644 --- a/pandas/io/tests/test_gbq.py +++ b/pandas/io/tests/test_gbq.py @@ -30,45 +30,44 @@ def _test_imports(): - if not compat.PY3: + global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \ + _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED - global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \ - _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED - - try: - import pkg_resources - _SETUPTOOLS_INSTALLED = True - except ImportError: - _SETUPTOOLS_INSTALLED = False - - if _SETUPTOOLS_INSTALLED: - try: - from apiclient.discovery import build - from apiclient.errors import HttpError + try: + import pkg_resources + _SETUPTOOLS_INSTALLED = True + except ImportError: + _SETUPTOOLS_INSTALLED = False - from oauth2client.client import OAuth2WebServerFlow - from oauth2client.client import AccessTokenRefreshError + if compat.PY3: + google_api_minimum_version = '1.4.1' + else: + google_api_minimum_version = '1.2.0' - from oauth2client.file import Storage - from oauth2client.tools import run_flow - _GOOGLE_API_CLIENT_INSTALLED=True - _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version + if _SETUPTOOLS_INSTALLED: + try: + from apiclient.discovery import build + from apiclient.errors import HttpError - if StrictVersion(_GOOGLE_API_CLIENT_VERSION) >= StrictVersion('1.2.0'): - _GOOGLE_API_CLIENT_VALID_VERSION = True + from oauth2client.client import OAuth2WebServerFlow + from oauth2client.client import AccessTokenRefreshError - except ImportError: - _GOOGLE_API_CLIENT_INSTALLED = False + from oauth2client.file import Storage + from oauth2client.tools import run_flow + _GOOGLE_API_CLIENT_INSTALLED=True + _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution('google-api-python-client').version + if StrictVersion(_GOOGLE_API_CLIENT_VERSION) >= StrictVersion(google_api_minimum_version): + _GOOGLE_API_CLIENT_VALID_VERSION = True - try: - import httplib2 - _HTTPLIB2_INSTALLED = True - except ImportError: - _HTTPLIB2_INSTALLED = False + except ImportError: + _GOOGLE_API_CLIENT_INSTALLED = False - if compat.PY3: - raise NotImplementedError("Google's libraries do not support Python 3 yet") + try: + import httplib2 + _HTTPLIB2_INSTALLED = True + except ImportError: + _HTTPLIB2_INSTALLED = False if not _SETUPTOOLS_INSTALLED: raise ImportError('Could not import pkg_resources (setuptools).') @@ -77,8 +76,8 @@ def _test_imports(): raise ImportError('Could not import Google API Client.') if not _GOOGLE_API_CLIENT_VALID_VERSION: - raise ImportError("pandas requires google-api-python-client >= 1.2.0 for Google " - "BigQuery support, current version " + _GOOGLE_API_CLIENT_VERSION) + raise ImportError("pandas requires google-api-python-client >= {0} for Google BigQuery support, " + "current version {1}".format(google_api_minimum_version, _GOOGLE_API_CLIENT_VERSION)) if not _HTTPLIB2_INSTALLED: raise ImportError("pandas requires httplib2 for Google BigQuery support") @@ -299,7 +298,12 @@ def test_unicode_string_conversion_and_normalization(self): {'UNICODE_STRING': [u("\xe9\xfc")]} ) - query = 'SELECT "\xc3\xa9\xc3\xbc" as UNICODE_STRING' + unicode_string = "\xc3\xa9\xc3\xbc" + + if compat.PY3: + unicode_string = unicode_string.encode('latin-1').decode('utf8') + + query = 'SELECT "{0}" as UNICODE_STRING'.format(unicode_string) df = gbq.read_gbq(query, project_id=PROJECT_ID) tm.assert_frame_equal(df, correct_test_datatype)