Use tuple for credentials & project for default project detection.

Author: tswast

pandas_gbq/gbq.py

@@ -189,7 +189,15 @@ def __init__(self, project_id, reauth=False,
         self.auth_local_webserver = auth_local_webserver
         self.dialect = dialect
         self.credentials_path = _get_credentials_file()
-        self.credentials = self.get_credentials()
+        self.credentials, default_project = self.get_credentials()
+
+        if self.project_id is None:
+            self.project_id = default_project
+
+        if self.project_id is None:
+            raise ValueError(
+                'Could not determine project ID and one was not supplied.')
+
         self.client = self.get_client()
 
         # BQ Queries costs $5 per TB. First 1 TB per month is free
@@ -199,12 +207,13 @@ def __init__(self, project_id, reauth=False,
     def get_credentials(self):
         if self.private_key:
             return self.get_service_account_credentials()
-        else:
-            # Try to retrieve Application Default Credentials
-            credentials = self.get_application_default_credentials()
-            if not credentials:
-                credentials = self.get_user_account_credentials()
-            return credentials
+
+        # Try to retrieve Application Default Credentials
+        credentials, default_project = self.get_application_default_credentials()
+        if credentials:
+            return credentials, default_project
+
+        return self.get_user_account_credentials(), None
 
     def get_application_default_credentials(self):
         """
@@ -230,11 +239,13 @@ def get_application_default_credentials(self):
         from google.auth.exceptions import DefaultCredentialsError
 
         try:
-            credentials, _ = google.auth.default(scopes=[self.scope])
+            credentials, default_project = google.auth.default(
+                scopes=[self.scope])
         except (DefaultCredentialsError, IOError):
-            return None
+            return None, None
 
-        return _try_credentials(self.project_id, credentials)
+        billing_project = self.project_id or default_project
+        return _try_credentials(billing_project, credentials), default_project
 
     def load_user_account_credentials(self):
         """
@@ -415,7 +426,7 @@ def get_service_account_credentials(self):
             request = google.auth.transport.requests.Request()
             credentials.refresh(request)
 
-            return credentials
+            return credentials, json_key.get('project_id')
         except (KeyError, ValueError, TypeError, AttributeError):
             raise InvalidPrivateKeyFormat(
                 "Private key is missing or invalid. It should be service "

tests/system.py

@@ -22,6 +22,9 @@
 
 TABLE_ID = 'new_test'
 
+def _skip_local_auth_if_in_travis_env():
+    if _in_travis_environment():
+        pytest.skip("Cannot run local auth in travis environment")
 
 def _skip_if_no_private_key_path():
     if not _get_private_key_path():
@@ -167,43 +170,13 @@ def test_generate_bq_schema_deprecated():
         gbq.generate_bq_schema(df)
 
 
-@pytest.fixture(params=[
-    pytest.param('local', marks=pytest.mark.local_auth),
-    pytest.param('service_path', marks=pytest.mark.s_path_auth),
-    pytest.param('service_creds', marks=pytest.mark.s_cred_auth),
-])
-def auth_type(request):
-
-    auth = request.param
-
-    if auth == 'local':
-        pass
-    elif auth == 'service_path':
-        _skip_if_no_private_key_path()
-    elif auth == 'service_creds':
-        _skip_if_no_private_key_contents()
-    else:
-        raise ValueError
-    return auth
-
-
 @pytest.fixture()
-def credentials(auth_type):
-
-    if auth_type == 'local':
-        return None
-
-    elif auth_type == 'service_path':
-        return _get_private_key_path()
-    elif auth_type == 'service_creds':
-        return _get_private_key_contents()
-    else:
-        raise ValueError
+def credentials():
+    return _get_private_key_contents()
 
 
 @pytest.fixture()
 def gbq_connector(project, credentials):
-
     return gbq.GbqConnector(project, private_key=credentials)
 
 
@@ -229,13 +202,12 @@ def test_should_be_able_to_get_results_from_query(self, gbq_connector):
         assert pages is not None
 
 
-@pytest.mark.local_auth
-class TestGBQConnectorIntegrationWithLocalUserAccountAuth(object):
+class TestAuth(object):
 
     @pytest.fixture(autouse=True)
-    def setup(self, project):
-
-        self.sut = gbq.GbqConnector(project, auth_local_webserver=True)
+    def setup(self, gbq_connector):
+        self.sut = gbq_connector
+        self.sut.auth_local_webserver = True
 
     def test_get_application_default_credentials_does_not_throw_error(self):
         if _check_if_can_get_correct_default_credentials():
@@ -244,27 +216,33 @@ def test_get_application_default_credentials_does_not_throw_error(self):
             from google.auth.exceptions import DefaultCredentialsError
             with mock.patch('google.auth.default',
                             side_effect=DefaultCredentialsError()):
-                credentials = self.sut.get_application_default_credentials()
+                credentials, _ = self.sut.get_application_default_credentials()
         else:
-            credentials = self.sut.get_application_default_credentials()
+            credentials, _ = self.sut.get_application_default_credentials()
         assert credentials is None
 
     def test_get_application_default_credentials_returns_credentials(self):
         if not _check_if_can_get_correct_default_credentials():
             pytest.skip("Cannot get default_credentials "
                         "from the environment!")
         from google.auth.credentials import Credentials
-        credentials = self.sut.get_application_default_credentials()
+        credentials, default_project = (
+            self.sut.get_application_default_credentials())
+
         assert isinstance(credentials, Credentials)
+        assert default_project is not None
 
     def test_get_user_account_credentials_bad_file_returns_credentials(self):
+        _skip_local_auth_if_in_travis_env()
 
         from google.auth.credentials import Credentials
         with mock.patch('__main__.open', side_effect=IOError()):
             credentials = self.sut.get_user_account_credentials()
         assert isinstance(credentials, Credentials)
 
     def test_get_user_account_credentials_returns_credentials(self):
+        _skip_local_auth_if_in_travis_env()
+
         from google.auth.credentials import Credentials
         credentials = self.sut.get_user_account_credentials()
         assert isinstance(credentials, Credentials)

tests/unit/test_gbq.py

@@ -15,21 +15,26 @@
 
 @pytest.fixture(autouse=True)
 def mock_bigquery_client(monkeypatch):
+    from google.api_core.exceptions import NotFound
     import google.cloud.bigquery
     import google.cloud.bigquery.table
     mock_client = mock.create_autospec(google.cloud.bigquery.Client)
+    mock_schema = [
+        google.cloud.bigquery.SchemaField('_f0', 'INTEGER')
+    ]
     # Mock out SELECT 1 query results.
     mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob)
     mock_query.job_id = 'some-random-id'
     mock_query.state = 'DONE'
     mock_rows = mock.create_autospec(
         google.cloud.bigquery.table.RowIterator)
     mock_rows.total_rows = 1
-    mock_rows.schema = [
-        google.cloud.bigquery.SchemaField('_f0', 'INTEGER')]
+    mock_rows.schema = mock_schema
     mock_rows.__iter__.return_value = [(1,)]
     mock_query.result.return_value = mock_rows
     mock_client.query.return_value = mock_query
+    # Mock table creation.
+    mock_client.get_table.side_effect = NotFound('nope')
     monkeypatch.setattr(
         gbq.GbqConnector, 'get_client', lambda _: mock_client)
 
@@ -42,11 +47,7 @@ def no_auth(monkeypatch):
     monkeypatch.setattr(
         gbq.GbqConnector,
         'get_application_default_credentials',
-        lambda _: mock_credentials)
-    monkeypatch.setattr(
-        gbq.GbqConnector,
-        'get_user_account_credentials',
-        lambda _: mock_credentials)
+        lambda _: (mock_credentials, 'default-project'))
 
 
 def test_should_return_credentials_path_set_by_env_var():
@@ -76,12 +77,16 @@ def test_should_return_bigquery_correctly_typed(
 
 def test_to_gbq_should_fail_if_invalid_table_name_passed():
     with pytest.raises(gbq.NotFoundException):
-        gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234")
+        gbq.to_gbq(DataFrame([[1]]), 'invalid_table_name', project_id="1234")
 
 
-def test_to_gbq_with_no_project_id_given_should_fail():
+def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch):
+    monkeypatch.setattr(
+        gbq.GbqConnector,
+        'get_application_default_credentials',
+        lambda _: None)
     with pytest.raises(TypeError):
-        gbq.to_gbq(DataFrame(), 'dataset.tablename')
+        gbq.to_gbq(DataFrame([[1]]), 'dataset.tablename')
 
 
 def test_to_gbq_with_verbose_new_pandas_warns_deprecation():
@@ -95,7 +100,7 @@ def test_to_gbq_with_verbose_new_pandas_warns_deprecation():
         mock_version.side_effect = [min_bq_version, pandas_version]
         try:
             gbq.to_gbq(
-                DataFrame(),
+                DataFrame([[1]]),
                 'dataset.tablename',
                 project_id='my-project',
                 verbose=True)
@@ -114,7 +119,7 @@ def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation():
         mock_version.side_effect = [min_bq_version, pandas_version]
         try:
             gbq.to_gbq(
-                DataFrame(),
+                DataFrame([[1]]),
                 'dataset.tablename',
                 project_id='my-project',
                 verbose=False)
@@ -132,7 +137,7 @@ def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn):
         mock_version.side_effect = [min_bq_version, pandas_version]
         try:
             gbq.to_gbq(
-                DataFrame(), 'dataset.tablename', project_id='my-project')
+                DataFrame([[1]]), 'dataset.tablename', project_id='my-project')
         except gbq.TableCreationError:
             pass
         assert len(recwarn) == 0
@@ -148,7 +153,7 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn):
         mock_version.side_effect = [min_bq_version, pandas_version]
         try:
             gbq.to_gbq(
-                DataFrame(),
+                DataFrame([[1]]),
                 'dataset.tablename',
                 project_id='my-project',
                 verbose=True)
@@ -157,11 +162,20 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn):
         assert len(recwarn) == 0
 
 
-def test_read_gbq_with_no_project_id_given_should_fail():
+def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch):
+    monkeypatch.setattr(
+        gbq.GbqConnector,
+        'get_application_default_credentials',
+        lambda _: None)
     with pytest.raises(TypeError):
         gbq.read_gbq('SELECT 1')
 
 
+def test_read_gbq_with_inferred_project_id(monkeypatch):
+    df = gbq.read_gbq('SELECT 1')
+    assert df is not None
+
+
 def test_that_parse_data_works_properly():
     from google.cloud.bigquery.table import Row
     test_schema = {'fields': [