Skip to content

Commit a1fdeb1

Browse files
tswasttswast
committed
use FEATURES
1 parent 762236d commit a1fdeb1

File tree

6 files changed

+54
-44
lines changed

6 files changed

+54
-44
lines changed

‎pandas_gbq/features.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
BIGQUERY_QUERY_AND_WAIT_VERSION = "3.14.0"
1010
PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0"
1111
PANDAS_BOOLEAN_DTYPE_VERSION = "1.0.0"
12+
PANDAS_MICROSECTONDS_DATETIME_VERSION = "2.1.0"
1213

1314

1415
class Features:
@@ -81,5 +82,12 @@ def pandas_has_boolean_dtype(self):
8182
desired_version = packaging.version.parse(PANDAS_BOOLEAN_DTYPE_VERSION)
8283
return self.pandas_installed_version >= desired_version
8384

85+
@property
86+
def pandas_has_microseconds_datetime(self):
87+
import packaging.version
88+
89+
desired_version = packaging.version.parse(PANDAS_MICROSECTONDS_DATETIME_VERSION)
90+
return self.pandas_installed_version >= desired_version
91+
8492

8593
FEATURES = Features()

‎pandas_gbq/gbq.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -639,7 +639,7 @@ def _finalize_dtypes(
639639
"DATETIME": "datetime64[ns]",
640640
"TIMESTAMP": "datetime64[ns]",
641641
}
642-
if tuple(int(part) for part in pandas.__version__.split(".")[:2]) >= (2, 1):
642+
if FEATURES.pandas_has_microseconds_datetime:
643643
# when pandas is 2.1.0 or later, default timestamp dtype is 'datetime64[us]'
644644
# and we should use 'datetime64[us]' instead of 'datetime64[ns]'
645645
dtype_map = {

‎pandas_gbq/load/core.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ def load_csv_from_dataframe(
195195
bq_schema = pandas_gbq.schema.to_google_cloud_bigquery(schema)
196196

197197
def load_chunk(chunk, job_config):
198+
breakpoint()
198199
client.load_table_from_dataframe(
199200
chunk,
200201
destination_table_ref,

‎tests/system/test_read_gbq.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
from pandas_gbq.features import FEATURES
1818

19+
1920
QueryTestCase = collections.namedtuple(
2021
"QueryTestCase",
2122
["query", "expected", "use_bqstorage_apis"],
@@ -628,7 +629,9 @@ def test_empty_dataframe(read_gbq, use_bqstorage_api):
628629
),
629630
"datetime_col": pandas.Series(
630631
[],
631-
dtype="datetime64[ns]",
632+
dtype="datetime64[us]"
633+
if FEATURES.pandas_has_microseconds_datetime
634+
else "datetime64[ns]",
632635
),
633636
"float_col": pandas.Series([], dtype="float64"),
634637
"int64_col": pandas.Series([], dtype="Int64"),
@@ -640,8 +643,10 @@ def test_empty_dataframe(read_gbq, use_bqstorage_api):
640643
),
641644
"timestamp_col": pandas.Series(
642645
[],
643-
dtype="datetime64[ns]",
644-
).dt.tz_localize(datetime.timezone.utc),
646+
dtype=pandas.DatetimeTZDtype(unit="us", tz="UTC")
647+
if FEATURES.pandas_has_microseconds_datetime
648+
else pandas.DatetimeTZDtype(tz="UTC"),
649+
),
645650
}
646651
)
647652
result = read_gbq(query, use_bqstorage_api=use_bqstorage_api)

‎tests/system/test_to_gbq.py

Lines changed: 30 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
pytest.importorskip("google.cloud.bigquery", minversion="1.24.0")
1717

1818

19+
PANDAS_VERSION = tuple(int(part) for part in pandas.__version__.split(".")[:2])
20+
21+
1922
@pytest.fixture(params=["load_parquet", "load_csv"])
2023
def api_method(request):
2124
return request.param
@@ -343,25 +346,33 @@ def test_series_round_trip(
343346
# require `date_as_object` parameter in
344347
# google-cloud-bigquery versions 1.x and 2.x, but not 3.x.
345348
# https://github.com/googleapis/python-bigquery-pandas/issues/365
346-
"datetime_col": [
347-
datetime.datetime(1, 1, 1),
348-
datetime.datetime(1970, 1, 1),
349-
datetime.datetime(9999, 12, 31, 23, 59, 59, 999999),
350-
],
351-
"timestamp_col": [
352-
datetime.datetime(1, 1, 1, tzinfo=datetime.timezone.utc),
353-
datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc),
354-
datetime.datetime(
355-
9999,
356-
12,
357-
31,
358-
23,
359-
59,
360-
59,
361-
999999,
362-
tzinfo=datetime.timezone.utc,
363-
),
364-
],
349+
"datetime_col": pandas.Series(
350+
[
351+
datetime.datetime(1, 1, 1),
352+
datetime.datetime(1970, 1, 1),
353+
datetime.datetime(9999, 12, 31, 23, 59, 59, 999999),
354+
],
355+
dtype="object" if PANDAS_VERSION < (2, 1) else "datetime64[us]",
356+
),
357+
"timestamp_col": pandas.Series(
358+
[
359+
datetime.datetime(1, 1, 1, tzinfo=datetime.timezone.utc),
360+
datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc),
361+
datetime.datetime(
362+
9999,
363+
12,
364+
31,
365+
23,
366+
59,
367+
59,
368+
999999,
369+
tzinfo=datetime.timezone.utc,
370+
),
371+
],
372+
dtype="object"
373+
if PANDAS_VERSION < (2, 1)
374+
else pandas.DatetimeTZDtype(unit="us", tz="UTC"),
375+
),
365376
},
366377
columns=["row_num", "date_col", "datetime_col", "timestamp_col"],
367378
),

‎tests/unit/test_gbq.py

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,6 @@
2828
pytestmark = pytest.mark.filterwarnings("ignore:credentials from Google Cloud SDK")
2929

3030

31-
PANDAS_VERSION = tuple(int(part) for part in pandas.__version__.split(".")[:2])
32-
33-
3431
def _make_connector(project_id="some-project", **kwargs):
3532
return gbq.GbqConnector(project_id, **kwargs)
3633

@@ -122,55 +119,43 @@ def test__bqschema_to_nullsafe_dtypes(type_, expected):
122119
pytest.param(
123120
pandas.to_datetime(["2017-01-01T12:00:00Z"]).astype(
124121
pandas.DatetimeTZDtype(
125-
# Microseconds aren't supported until newer pandas.
126-
# https://github.com/googleapis/python-bigquery-pandas/issues/852
127-
unit="us" if PANDAS_VERSION >= (2, 1) else "ns",
122+
unit="us" if FEATURES.pandas_has_microseconds_datetime else "ns",
128123
tz="UTC",
129124
),
130125
),
131126
"TIMESTAMP",
132127
pandas.DatetimeTZDtype(
133-
# Microseconds aren't supported until newer pandas.
134-
# https://github.com/googleapis/python-bigquery-pandas/issues/852
135-
unit="us" if PANDAS_VERSION >= (2, 1) else "ns",
128+
unit="us" if FEATURES.pandas_has_microseconds_datetime else "ns",
136129
tz="UTC",
137130
),
138131
),
139132
(
140133
pandas.to_datetime([]).astype(object),
141134
"TIMESTAMP",
142135
pandas.DatetimeTZDtype(
143-
# Microseconds aren't supported until newer pandas.
144-
# https://github.com/googleapis/python-bigquery-pandas/issues/852
145-
unit="us" if PANDAS_VERSION >= (2, 1) else "ns",
136+
unit="us" if FEATURES.pandas_has_microseconds_datetime else "ns",
146137
tz="UTC",
147138
),
148139
),
149140
(
150141
pandas.to_datetime(["2017-01-01T12:00:00"]).astype(
151-
# Microseconds aren't supported until newer pandas.
152-
# https://github.com/googleapis/python-bigquery-pandas/issues/852
153142
"datetime64[us]"
154-
if PANDAS_VERSION >= (2, 1)
143+
if FEATURES.pandas_has_microseconds_datetime
155144
else "datetime64[ns]",
156145
),
157146
"DATETIME",
158147
numpy.dtype(
159-
# Microseconds aren't supported until newer pandas.
160-
# https://github.com/googleapis/python-bigquery-pandas/issues/852
161148
"datetime64[us]"
162-
if PANDAS_VERSION >= (2, 1)
149+
if FEATURES.pandas_has_microseconds_datetime
163150
else "datetime64[ns]",
164151
),
165152
),
166153
(
167154
pandas.to_datetime([]).astype(object),
168155
"DATETIME",
169156
numpy.dtype(
170-
# Microseconds aren't supported until newer pandas.
171-
# https://github.com/googleapis/python-bigquery-pandas/issues/852
172157
"datetime64[us]"
173-
if PANDAS_VERSION >= (2, 1)
158+
if FEATURES.pandas_has_microseconds_datetime
174159
else "datetime64[ns]",
175160
),
176161
),

0 commit comments

Comments
 (0)