support google-cloud-bigquery 3.x (test with pre-release dependencies)

[tswast]

There is currently a beta 1 pre-release for google-cloud-bigquery 3.x, it'd be great if we could expand our support for it already -- or at least run some tests to see what breaks.

It does change some default dtypes, so there is a good chance at least some tests will fail.

[tswast]

Failing tests with google-cloud-bigquery==3.0.0b1

nox > Running session prerelease-3.7
nox > Creating virtual environment (virtualenv) using python3.7 in .nox/prerelease-3-7
nox > python -m pip install --extra-index-url https://pypi.fury.io/arrow-nightlies/ --prefer-binary --pre --upgrade pyarrow
nox > python -m pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple --prefer-binary --pre --upgrade pandas
nox > python -m pip install --prefer-binary --pre --upgrade google-api-core google-cloud-bigquery google-cloud-bigquery-storage google-cloud-core google-resumable-media grpcio
nox > python -m pip install freezegun google-cloud-datacatalog google-cloud-storage google-cloud-testutils IPython mock psutil pytest pytest-cov
nox > python -m pip install db-dtypes google-auth google-auth-oauthlib google-cloud-bigquery google-cloud-bigquery-storage numpy pandas pyarrow pydata-google-auth tqdm
nox > python -m pip install --no-deps -e .[all]
nox > python -m pip freeze
appnope==0.1.2
attrs==21.2.0
backcall==0.2.0
cachetools==4.2.4
certifi==2021.10.8
charset-normalizer==2.0.8
click==8.0.3
coverage==6.2
db-dtypes==0.3.0
decorator==5.1.0
freezegun==1.1.0
google-api-core==2.2.2
google-auth==2.3.3
google-auth-oauthlib==0.4.6
google-cloud-bigquery==3.0.0b1
google-cloud-bigquery-storage==2.10.1
google-cloud-core==2.2.1
google-cloud-datacatalog==3.6.1
google-cloud-storage==1.43.0
google-cloud-testutils==1.2.0
google-crc32c==1.3.0
google-resumable-media==2.1.0
googleapis-common-protos==1.53.0
grpc-google-iam-v1==0.12.3
grpcio==1.42.0
grpcio-status==1.42.0
idna==3.3
importlib-metadata==4.8.2
iniconfig==1.1.1
ipython==7.30.0
jedi==0.18.1
libcst==0.3.23
matplotlib-inline==0.1.3
mock==4.0.3
mypy-extensions==0.4.3
numpy==1.21.4
oauthlib==3.1.1
packaging==21.3
pandas==1.4.0.dev0+143.g5675cd8ab2
-e git+ssh://git@github.com/tswast/python-bigquery-pandas.git@21808367d02b5b7fcf35b3c7520224c819879aec#egg=pandas_gbq
parso==0.8.2
pexpect==4.8.0
pickleshare==0.7.5
pluggy==1.0.0
prompt-toolkit==3.0.23
proto-plus==1.19.8
protobuf==3.19.1
psutil==5.8.0
ptyprocess==0.7.0
py==1.11.0
pyarrow==6.0.1
pyasn1==0.4.8
pyasn1-modules==0.2.8
pydata-google-auth==1.2.0
Pygments==2.10.0
pyparsing==3.0.6
pytest==6.2.5
pytest-cov==3.0.0
python-dateutil==2.8.2
pytz==2021.3
PyYAML==6.0
requests==2.26.0
requests-oauthlib==1.3.0
rsa==4.8
six==1.16.0
toml==0.10.2
tomli==1.2.2
tqdm==4.62.3
traitlets==5.1.1
typing-inspect==0.7.1
typing_extensions==4.0.0
urllib3==1.26.7
wcwidth==0.2.5
zipp==3.6.0
nox > py.test --quiet --junitxml=prerelease_unit_3.7_sponge_log.xml tests/unit
..........................................................................................                             [100%]
---- generated xml file: /Users/swast/src/github.com/googleapis/python-bigquery-pandas/prerelease_unit_3.7_sponge_log.xml ----
90 passed in 17.88s
nox > py.test --quiet --junitxml=prerelease_system_3.7_sponge_log.xml tests/system
.........F.F....................FF..............F..F.....F.F.F.........s..................................s....s...    [100%]
========================================================== FAILURES ==========================================================
_____________________________ TestReadGBQIntegration.test_should_properly_handle_valid_integers ______________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7f9fc87bfa50>, project_id = 'swast-scratch'

    def test_should_properly_handle_valid_integers(self, project_id):
        query = "SELECT CAST(3 AS INT64) AS valid_integer"
        df = gbq.read_gbq(
            query,
            project_id=project_id,
            credentials=self.credentials,
            dialect="standard",
        )
>       tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]}))
E       AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="valid_integer") are different
E       
E       Attribute "dtype" are different
E       [left]:  Int64
E       [right]: int64

tests/system/test_gbq.py:171: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 1/1 [00:00<00:00,  4.62rows/s]
_______________________________ TestReadGBQIntegration.test_should_properly_handle_valid_longs _______________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7fa008649c50>, project_id = 'swast-scratch'

    def test_should_properly_handle_valid_longs(self, project_id):
        query = "SELECT 1 << 62 AS valid_long"
        df = gbq.read_gbq(
            query,
            project_id=project_id,
            credentials=self.credentials,
            dialect="standard",
        )
>       tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]}))
E       AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="valid_long") are different
E       
E       Attribute "dtype" are different
E       [left]:  Int64
E       [right]: int64

tests/system/test_gbq.py:197: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 1/1 [00:00<00:00,  3.91rows/s]
______________________________ TestReadGBQIntegration.test_should_properly_handle_null_boolean _______________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7f9fb83bc710>, project_id = 'swast-scratch'

    def test_should_properly_handle_null_boolean(self, project_id):
        query = "SELECT BOOLEAN(NULL) AS null_boolean"
        df = gbq.read_gbq(
            query,
            project_id=project_id,
            credentials=self.credentials,
            dialect="legacy",
        )
>       tm.assert_frame_equal(df, DataFrame({"null_boolean": [None]}))
E       AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="null_boolean") are different
E       
E       Attribute "dtype" are different
E       [left]:  boolean
E       [right]: object

tests/system/test_gbq.py:436: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 1/1 [00:00<00:00,  3.63rows/s]
____________________________ TestReadGBQIntegration.test_should_properly_handle_nullable_booleans ____________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7f9fb83c5a50>, project_id = 'swast-scratch'

    def test_should_properly_handle_nullable_booleans(self, project_id):
        query = """SELECT * FROM
                    (SELECT BOOLEAN(TRUE) AS nullable_boolean),
                    (SELECT NULL AS nullable_boolean)"""
        df = gbq.read_gbq(
            query,
            project_id=project_id,
            credentials=self.credentials,
            dialect="legacy",
        )
        tm.assert_frame_equal(
>           df, DataFrame({"nullable_boolean": [True, None]}).astype(object)
        )
E       AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="nullable_boolean") are different
E       
E       Attribute "dtype" are different
E       [left]:  boolean
E       [right]: object

tests/system/test_gbq.py:449: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 2/2 [00:00<00:00,  9.62rows/s]
_______________________________________ TestReadGBQIntegration.test_one_row_one_column _______________________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7f9fb83a8ad0>, project_id = 'swast-scratch'

    def test_one_row_one_column(self, project_id):
        df = gbq.read_gbq(
            "SELECT 3 as v",
            project_id=project_id,
            credentials=self.credentials,
            dialect="standard",
        )
        expected_result = DataFrame(dict(v=[3]))
>       tm.assert_frame_equal(df, expected_result)
E       AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="v") are different
E       
E       Attribute "dtype" are different
E       [left]:  Int64
E       [right]: int64

tests/system/test_gbq.py:633: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 1/1 [00:00<00:00,  4.90rows/s]
_____________________________________ TestReadGBQIntegration.test_query_with_parameters ______________________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7f9fb8387710>, project_id = 'swast-scratch'

    def test_query_with_parameters(self, project_id):
        sql_statement = "SELECT @param1 + @param2 AS valid_result"
        config = {
            "query": {
                "useLegacySql": False,
                "parameterMode": "named",
                "queryParameters": [
                    {
                        "name": "param1",
                        "parameterType": {"type": "INTEGER"},
                        "parameterValue": {"value": 1},
                    },
                    {
                        "name": "param2",
                        "parameterType": {"type": "INTEGER"},
                        "parameterValue": {"value": 2},
                    },
                ],
            }
        }
        # Test that a query that relies on parameters fails
        # when parameters are not supplied via configuration
        with pytest.raises(ValueError):
            gbq.read_gbq(
                sql_statement,
                project_id=project_id,
                credentials=self.credentials,
                dialect="legacy",
            )
    
        # Test that the query is successful because we have supplied
        # the correct query parameters via the 'config' option
        df = gbq.read_gbq(
            sql_statement,
            project_id=project_id,
            credentials=self.credentials,
            configuration=config,
            dialect="legacy",
        )
>       tm.assert_frame_equal(df, DataFrame({"valid_result": [3]}))
E       AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="valid_result") are different
E       
E       Attribute "dtype" are different
E       [left]:  Int64
E       [right]: int64

tests/system/test_gbq.py:722: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 1/1 [00:00<00:00,  5.06rows/s]
_____________________________________________ TestReadGBQIntegration.test_struct _____________________________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7f9fb8378290>, project_id = 'swast-scratch'

    def test_struct(self, project_id):
        query = """SELECT 1 int_field,
                   STRUCT("a" as letter, 1 as num) struct_field"""
        df = gbq.read_gbq(
            query,
            project_id=project_id,
            credentials=self.credentials,
            dialect="standard",
        )
        expected = DataFrame(
            [[1, {"letter": "a", "num": 1}]], columns=["int_field", "struct_field"],
        )
>       tm.assert_frame_equal(df, expected)
E       AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="int_field") are different
E       
E       Attribute "dtype" are different
E       [left]:  Int64
E       [right]: int64

tests/system/test_gbq.py:847: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 1/1 [00:00<00:00,  5.32rows/s]
_______________________________________ TestReadGBQIntegration.test_array_length_zero ________________________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7f9fc87d1910>, project_id = 'swast-scratch'

    def test_array_length_zero(self, project_id):
        query = """WITH t as (
                   SELECT "a" letter, [""] as array_field
                   UNION ALL
                   SELECT "b" letter, [] as array_field)
    
                   select letter, array_field, array_length(array_field) len
                   from t
                   order by letter ASC"""
        df = gbq.read_gbq(
            query,
            project_id=project_id,
            credentials=self.credentials,
            dialect="standard",
        )
        expected = DataFrame(
            [["a", [""], 1], ["b", [], 0]], columns=["letter", "array_field", "len"],
        )
>       tm.assert_frame_equal(df, expected)
E       AssertionError: Attributes of DataFrame.iloc[:, 2] (column name="len") are different
E       
E       Attribute "dtype" are different
E       [left]:  Int64
E       [right]: int64

tests/system/test_gbq.py:879: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 2/2 [00:00<00:00,  9.53rows/s]
________________________________________ TestReadGBQIntegration.test_array_of_floats _________________________________________

self = <system.test_gbq.TestReadGBQIntegration object at 0x7f9fb83a8310>, project_id = 'swast-scratch'

    def test_array_of_floats(self, project_id):
        query = """select [1.1, 2.2, 3.3] as a, 4 as b"""
        df = gbq.read_gbq(
            query,
            project_id=project_id,
            credentials=self.credentials,
            dialect="standard",
        )
>       tm.assert_frame_equal(df, DataFrame([[[1.1, 2.2, 3.3], 4]], columns=["a", "b"]))
E       AssertionError: Attributes of DataFrame.iloc[:, 1] (column name="b") are different
E       
E       Attribute "dtype" are different
E       [left]:  Int64
E       [right]: int64

tests/system/test_gbq.py:911: AssertionError
---------------------------------------------------- Captured stderr call ----------------------------------------------------
Downloading: 100%|██████████| 1/1 [00:00<00:00,  4.82rows/s]
====================================================== warnings summary ======================================================
tests/system/test_gbq.py:13
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/tests/system/test_gbq.py:13: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
    import pandas.util.testing as tm

tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_timestamp_unix_epoch
tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_arbitrary_timestamp
tests/system/test_gbq.py::TestReadGBQIntegration::test_return_correct_types[current_timestamp()-is_datetime64tz_dtype]
tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_null_timestamp
tests/system/test_gbq.py::TestToGBQIntegration::test_upload_data_with_timestamp
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/.nox/prerelease-3-7/lib/python3.7/site-packages/google/cloud/bigquery/table.py:1947: FutureWarning: Using .astype to convert from timezone-aware dtype to timezone-naive dtype is deprecated and will raise in a future version.  Use obj.tz_localize(None) or obj.tz_convert('UTC').tz_localize(None) instead
    df[column] = pandas.Series(df[column], dtype=dtypes[column], copy=False)

tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_timestamp_unix_epoch
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/tests/system/test_gbq.py:329: FutureWarning: Data is timezone-aware. Converting timezone-aware data to timezone-naive by passing dtype='datetime64[ns]' to DataFrame or Series is deprecated and will raise in a future version. Use `pd.Series(values).dt.tz_localize(None)` instead.
    {"unix_epoch": ["1970-01-01T00:00:00.000000Z"]}, dtype="datetime64[ns]",

tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_arbitrary_timestamp
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/tests/system/test_gbq.py:345: FutureWarning: Data is timezone-aware. Converting timezone-aware data to timezone-naive by passing dtype='datetime64[ns]' to DataFrame or Series is deprecated and will raise in a future version. Use `pd.Series(values).dt.tz_localize(None)` instead.
    dtype="datetime64[ns]",

tests/system/test_gbq.py::TestToGBQIntegration::test_upload_data
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/tests/system/test_gbq.py:948: DeprecationWarning: chunksize is ignored when using api_method='load_parquet'
    credentials=self.credentials,

tests/system/test_gbq.py::TestToGBQIntegration::test_upload_data_if_table_exists_append
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/tests/system/test_gbq.py:1032: DeprecationWarning: chunksize is ignored when using api_method='load_parquet'
    credentials=self.credentials,

tests/system/test_gbq.py::TestToGBQIntegration::test_upload_subset_columns_if_table_exists_append
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/tests/system/test_gbq.py:1078: DeprecationWarning: chunksize is ignored when using api_method='load_parquet'
    credentials=self.credentials,

tests/system/test_gbq.py::TestToGBQIntegration::test_upload_data_if_table_exists_replace
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/tests/system/test_gbq.py:1112: DeprecationWarning: chunksize is ignored when using api_method='load_parquet'
    credentials=self.credentials,

tests/system/test_gbq.py::TestToGBQIntegration::test_upload_data_flexible_column_order
  /Users/swast/src/github.com/googleapis/python-bigquery-pandas/tests/system/test_gbq.py:1241: DeprecationWarning: chunksize is ignored when using api_method='load_parquet'
    credentials=self.credentials,

-- Docs: https://docs.pytest.org/en/stable/warnings.html
--- generated xml file: /Users/swast/src/github.com/googleapis/python-bigquery-pandas/prerelease_system_3.7_sponge_log.xml ---
================================================== short test summary info ===================================================
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_valid_integers - AssertionError: Attri...
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_valid_longs - AssertionError: Attribut...
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_null_boolean - AssertionError: Attribu...
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_should_properly_handle_nullable_booleans - AssertionError: At...
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_one_row_one_column - AssertionError: Attributes of DataFrame....
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_query_with_parameters - AssertionError: Attributes of DataFra...
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_struct - AssertionError: Attributes of DataFrame.iloc[:, 0] (...
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_array_length_zero - AssertionError: Attributes of DataFrame.i...
FAILED tests/system/test_gbq.py::TestReadGBQIntegration::test_array_of_floats - AssertionError: Attributes of DataFrame.ilo...
9 failed, 103 passed, 3 skipped, 13 warnings in 478.66s (0:07:58)
nox > Command py.test --quiet --junitxml=prerelease_system_3.7_sponge_log.xml tests/system failed with exit code 1
nox > Session prerelease-3.7 failed.

Edit: Source code for prerelease session:

@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
def prerelease(session):
    """Run all tests with prerelease versions of dependencies installed.
    https://github.com/googleapis/python-bigquery-pandas/issues/426
    """

    session.install(
        "--extra-index-url",
        "https://pypi.fury.io/arrow-nightlies/",
        "--prefer-binary",
        "--pre",
        "--upgrade",
        "pyarrow",
    )
    session.install(
        "--extra-index-url",
        "https://pypi.anaconda.org/scipy-wheels-nightly/simple",
        "--prefer-binary",
        "--pre",
        "--upgrade",
        "pandas",
    )
    session.install(
        "--prefer-binary",
        "--pre",
        "--upgrade",
        "google-api-core",
        "google-cloud-bigquery",
        "google-cloud-bigquery-storage",
        "google-cloud-core",
        "google-resumable-media",
        "grpcio",
    )
    session.install(
        "freezegun",
        "google-cloud-datacatalog",
        "google-cloud-storage",
        "google-cloud-testutils",
        "IPython",
        "mock",
        "psutil",
        "pytest",
        "pytest-cov",
    )

    # Because we test minimum dependency versions on the minimum Python
    # version, the first version we test with in the unit tests sessions has a
    # constraints file containing all dependencies and extras.
    with open(
        CURRENT_DIRECTORY
        / "testing"
        / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt",
        encoding="utf-8",
    ) as constraints_file:
        constraints_text = constraints_file.read()

    # Ignore leading whitespace and comment lines.
    deps = [
        match.group(1)
        for match in re.finditer(
            r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
        )
    ]

    # We use --no-deps to ensure that pre-release versions aren't overwritten
    # by the version ranges in setup.py.
    session.install(*deps)
    session.install("--no-deps", "-e", ".[all]")

    # Print out prerelease package versions.
    session.run("python", "-m", "pip", "freeze")

    # Run all tests, except a few samples tests which require extra dependencies.
    session.run(
        "py.test",
        "--quiet",
        f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml",
        os.path.join("tests", "unit"),
    )
    session.run(
        "py.test",
        "--quiet",
        f"--junitxml=prerelease_system_{session.python}_sponge_log.xml",
        os.path.join("tests", "system"),
    )

[tswast]

Look like just dtypes differences. I think we can actually implement this in pandas-gbq so that it behaves the same with google-cloud-bigquery 2.x and 3.x (preferring 3.x's extension dtypes where available).