Skip to content

Commit 38ac28d

Browse files
tswasttswast
authored
fix: address failing tests with pandas 1.5.0 (#82)
test: add a test session with prerelease versions of dependencies Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-db-dtypes-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes #81 🦕
1 parent f79466c commit 38ac28d

File tree

8 files changed

+314
-8
lines changed

8 files changed

+314
-8
lines changed

‎.github/workflows/compliance.yml

+21
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,24 @@ jobs:
2525
COVERAGE_FILE: .coverage-compliance-${{ matrix.python }}
2626
run: |
2727
nox -s compliance
28+
compliance-prerelease:
29+
runs-on: ubuntu-latest
30+
strategy:
31+
matrix:
32+
python: ['3.10']
33+
steps:
34+
- name: Checkout
35+
uses: actions/checkout@v3
36+
- name: Setup Python
37+
uses: actions/setup-python@v3
38+
with:
39+
python-version: ${{ matrix.python }}
40+
- name: Install nox
41+
run: |
42+
python -m pip install --upgrade setuptools pip wheel
43+
python -m pip install nox
44+
- name: Run compliance prerelease tests
45+
env:
46+
COVERAGE_FILE: .coverage-compliance-prerelease-${{ matrix.python }}
47+
run: |
48+
nox -s compliance_prerelease

‎.github/workflows/unittest-prerelease.yml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
on:
2+
pull_request:
3+
branches:
4+
- main
5+
name: unittest-prerelease
6+
jobs:
7+
unit:
8+
runs-on: ubuntu-latest
9+
strategy:
10+
matrix:
11+
python: ['3.10']
12+
steps:
13+
- name: Checkout
14+
uses: actions/checkout@v3
15+
- name: Setup Python
16+
uses: actions/setup-python@v3
17+
with:
18+
python-version: ${{ matrix.python }}
19+
- name: Install nox
20+
run: |
21+
python -m pip install --upgrade setuptools pip wheel
22+
python -m pip install nox
23+
- name: Run unit tests
24+
env:
25+
COVERAGE_FILE: .coverage-prerelease-${{ matrix.python }}
26+
run: |
27+
nox -s unit_prerelease
28+
- name: Upload coverage results
29+
uses: actions/upload-artifact@v3
30+
with:
31+
name: coverage-artifacts
32+
path: .coverage-${{ matrix.python }}

‎db_dtypes/__init__.py

+19-7
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@
3434
time_dtype_name = "dbtime"
3535
_EPOCH = datetime.datetime(1970, 1, 1)
3636
_NPEPOCH = numpy.datetime64(_EPOCH)
37+
_NP_DTYPE = "datetime64[ns]"
38+
39+
# Numpy converts datetime64 scalars to datetime.datetime only if microsecond or
40+
# smaller precision is used.
41+
#
42+
# TODO(https://github.com/googleapis/python-db-dtypes-pandas/issues/63): Keep
43+
# nanosecond precision when boxing scalars.
44+
_NP_BOX_DTYPE = "datetime64[us]"
3745

3846
pandas_release = packaging.version.parse(pandas.__version__).release
3947

@@ -149,12 +157,14 @@ def _box_func(self, x):
149157
return pandas.NaT
150158

151159
try:
152-
return x.astype("<M8[us]").astype(datetime.datetime).time()
160+
return x.astype(_NP_BOX_DTYPE).item().time()
153161
except AttributeError:
154-
x = numpy.datetime64(x)
155-
return x.astype("<M8[us]").astype(datetime.datetime).time()
162+
x = numpy.datetime64(
163+
x, "ns"
164+
) # Integers are stored with nanosecond precision.
165+
return x.astype(_NP_BOX_DTYPE).item().time()
156166

157-
__return_deltas = {"timedelta", "timedelta64", "timedelta64[ns]", "<m8", "<m8[ns]"}
167+
__return_deltas = {"timedelta", "timedelta64", "timedelta64[ns]", "<m8", _NP_DTYPE}
158168

159169
def astype(self, dtype, copy=True):
160170
deltas = self._ndarray - _NPEPOCH
@@ -253,10 +263,12 @@ def _box_func(self, x):
253263
if pandas.isna(x):
254264
return pandas.NaT
255265
try:
256-
return x.astype("<M8[us]").astype(datetime.datetime).date()
266+
return x.astype(_NP_BOX_DTYPE).item().date()
257267
except AttributeError:
258-
x = numpy.datetime64(x)
259-
return x.astype("<M8[us]").astype(datetime.datetime).date()
268+
x = numpy.datetime64(
269+
x, "ns"
270+
) # Integers are stored with nanosecond precision.
271+
return x.astype(_NP_BOX_DTYPE).item().date()
260272

261273
def astype(self, dtype, copy=True):
262274
stype = str(dtype)

‎db_dtypes/core.py

+6
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ def construct_from_string(cls, name: str):
4646
class BaseDatetimeArray(
4747
pandas_backports.OpsMixin, pandas_backports.NDArrayBackedExtensionArray
4848
):
49+
# scalar used to denote NA value inside our self._ndarray, e.g. -1 for
50+
# Categorical, iNaT for Period. Outside of object dtype, self.isna() should
51+
# be exactly locations in self._ndarray with _internal_fill_value. See:
52+
# https://github.com/pandas-dev/pandas/blob/main/pandas/core/arrays/_mixins.py
53+
_internal_fill_value = numpy.datetime64("NaT")
54+
4955
def __init__(self, values, dtype=None, copy: bool = False):
5056
if not (
5157
isinstance(values, numpy.ndarray) and values.dtype == numpy.dtype("<M8[ns]")

‎noxfile.py

+91
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from __future__ import absolute_import
2020
import os
2121
import pathlib
22+
import re
2223
import shutil
2324

2425
import nox
@@ -37,7 +38,9 @@
3738
nox.options.sessions = [
3839
"lint",
3940
"unit",
41+
"unit_prerelease",
4042
"compliance",
43+
"compliance_prerelease",
4144
"cover",
4245
"lint_setup_py",
4346
"blacken",
@@ -112,18 +115,106 @@ def default(session, tests_path):
112115
)
113116

114117

118+
def prerelease(session, tests_path):
119+
constraints_path = str(
120+
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
121+
)
122+
123+
# PyArrow prerelease packages are published to an alternative PyPI host.
124+
# https://arrow.apache.org/docs/python/install.html#installing-nightly-packages
125+
session.install(
126+
"--extra-index-url",
127+
"https://pypi.fury.io/arrow-nightlies/",
128+
"--prefer-binary",
129+
"--pre",
130+
"--upgrade",
131+
"pyarrow",
132+
)
133+
session.install(
134+
"--extra-index-url",
135+
"https://pypi.anaconda.org/scipy-wheels-nightly/simple",
136+
"--prefer-binary",
137+
"--pre",
138+
"--upgrade",
139+
"pandas",
140+
)
141+
session.install(
142+
"mock",
143+
"asyncmock",
144+
"pytest",
145+
"pytest-cov",
146+
"pytest-asyncio",
147+
"-c",
148+
constraints_path,
149+
)
150+
151+
# Because we test minimum dependency versions on the minimum Python
152+
# version, the first version we test with in the unit tests sessions has a
153+
# constraints file containing all dependencies and extras.
154+
with open(
155+
CURRENT_DIRECTORY
156+
/ "testing"
157+
/ f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt",
158+
encoding="utf-8",
159+
) as constraints_file:
160+
constraints_text = constraints_file.read()
161+
162+
# Ignore leading whitespace and comment lines.
163+
deps = [
164+
match.group(1)
165+
for match in re.finditer(
166+
r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
167+
)
168+
]
169+
170+
# We use --no-deps to ensure that pre-release versions aren't overwritten
171+
# by the version ranges in setup.py.
172+
session.install(*deps)
173+
session.install("--no-deps", "-e", ".")
174+
175+
# Print out prerelease package versions.
176+
session.run("python", "-m", "pip", "freeze")
177+
178+
# Run py.test against the unit tests.
179+
session.run(
180+
"py.test",
181+
"--quiet",
182+
f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml",
183+
"--cov=db_dtypes",
184+
"--cov=tests/unit",
185+
"--cov-append",
186+
"--cov-config=.coveragerc",
187+
"--cov-report=",
188+
"--cov-fail-under=0",
189+
tests_path,
190+
*session.posargs,
191+
)
192+
193+
115194
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
116195
def compliance(session):
117196
"""Run the compliance test suite."""
118197
default(session, os.path.join("tests", "compliance"))
119198

120199

200+
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
201+
def compliance_prerelease(session):
202+
"""Run the compliance test suite with prerelease dependencies."""
203+
prerelease(session, os.path.join("tests", "compliance"))
204+
205+
121206
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
122207
def unit(session):
123208
"""Run the unit test suite."""
124209
default(session, os.path.join("tests", "unit"))
125210

126211

212+
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
213+
def unit_prerelease(session):
214+
"""Run the unit test suite with prerelease dependencies."""
215+
prerelease(session, os.path.join("tests", "unit"))
216+
217+
127218
@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
128219
def system(session):
129220
"""Run the system test suite."""

‎owlbot.py

+95-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@
4949
["noxfile.py"], r"[\"']google[\"']", '"db_dtypes"',
5050
)
5151

52+
s.replace(
53+
["noxfile.py"], r"import shutil", "import re\nimport shutil",
54+
)
55+
5256
s.replace(
5357
["noxfile.py"], "--cov=google", "--cov=db_dtypes",
5458
)
@@ -64,7 +68,9 @@
6468
new_sessions = """
6569
"lint",
6670
"unit",
71+
"unit_prerelease",
6772
"compliance",
73+
"compliance_prerelease",
6874
"cover",
6975
"""
7076

@@ -83,17 +89,105 @@ def unit\(session\):
8389
"""Run the unit test suite."""
8490
default\(session\)
8591
''',
86-
'''
92+
r'''
93+
def prerelease(session, tests_path):
94+
constraints_path = str(
95+
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
96+
)
97+
98+
# PyArrow prerelease packages are published to an alternative PyPI host.
99+
# https://arrow.apache.org/docs/python/install.html#installing-nightly-packages
100+
session.install(
101+
"--extra-index-url",
102+
"https://pypi.fury.io/arrow-nightlies/",
103+
"--prefer-binary",
104+
"--pre",
105+
"--upgrade",
106+
"pyarrow",
107+
)
108+
session.install(
109+
"--extra-index-url",
110+
"https://pypi.anaconda.org/scipy-wheels-nightly/simple",
111+
"--prefer-binary",
112+
"--pre",
113+
"--upgrade",
114+
"pandas",
115+
)
116+
session.install(
117+
"mock",
118+
"asyncmock",
119+
"pytest",
120+
"pytest-cov",
121+
"pytest-asyncio",
122+
"-c",
123+
constraints_path,
124+
)
125+
126+
# Because we test minimum dependency versions on the minimum Python
127+
# version, the first version we test with in the unit tests sessions has a
128+
# constraints file containing all dependencies and extras.
129+
with open(
130+
CURRENT_DIRECTORY
131+
/ "testing"
132+
/ f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt",
133+
encoding="utf-8",
134+
) as constraints_file:
135+
constraints_text = constraints_file.read()
136+
137+
# Ignore leading whitespace and comment lines.
138+
deps = [
139+
match.group(1)
140+
for match in re.finditer(
141+
r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE
142+
)
143+
]
144+
145+
# We use --no-deps to ensure that pre-release versions aren't overwritten
146+
# by the version ranges in setup.py.
147+
session.install(*deps)
148+
session.install("--no-deps", "-e", ".")
149+
150+
# Print out prerelease package versions.
151+
session.run("python", "-m", "pip", "freeze")
152+
153+
# Run py.test against the unit tests.
154+
session.run(
155+
"py.test",
156+
"--quiet",
157+
f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml",
158+
"--cov=db_dtypes",
159+
"--cov=tests/unit",
160+
"--cov-append",
161+
"--cov-config=.coveragerc",
162+
"--cov-report=",
163+
"--cov-fail-under=0",
164+
tests_path,
165+
*session.posargs,
166+
)
167+
168+
87169
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
88170
def compliance(session):
89171
"""Run the compliance test suite."""
90172
default(session, os.path.join("tests", "compliance"))
91173
92174
175+
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
176+
def compliance_prerelease(session):
177+
"""Run the compliance test suite with prerelease dependencies."""
178+
prerelease(session, os.path.join("tests", "compliance"))
179+
180+
93181
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
94182
def unit(session):
95183
"""Run the unit test suite."""
96184
default(session, os.path.join("tests", "unit"))
185+
186+
187+
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
188+
def unit_prerelease(session):
189+
"""Run the unit test suite with prerelease dependencies."""
190+
prerelease(session, os.path.join("tests", "unit"))
97191
''',
98192
)
99193

0 commit comments

Comments
 (0)