fix: address failing tests with pandas 1.5.0

.github/workflows/compliance.yml

@@ -25,3 +25,24 @@ jobs:
         COVERAGE_FILE: .coverage-compliance-${{ matrix.python }}
       run: |
         nox -s compliance
+  compliance-prerelease:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ['3.10']
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install nox
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install nox
+    - name: Run compliance prerelease tests
+      env:
+        COVERAGE_FILE: .coverage-compliance-prerelease-${{ matrix.python }}
+      run: |
+        nox -s compliance_prerelease

.github/workflows/unittest-prerelease.yml

@@ -0,0 +1,32 @@
+on:
+  pull_request:
+    branches:
+      - main
+name: unittest-prerelease
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ['3.10']
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install nox
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install nox
+    - name: Run unit tests
+      env:
+        COVERAGE_FILE: .coverage-prerelease-${{ matrix.python }}
+      run: |
+        nox -s unit_prerelease
+    - name: Upload coverage results
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage-artifacts
+        path: .coverage-${{ matrix.python }}

db_dtypes/__init__.py

@@ -34,6 +34,14 @@
 time_dtype_name = "dbtime"
 _EPOCH = datetime.datetime(1970, 1, 1)
 _NPEPOCH = numpy.datetime64(_EPOCH)
+_NP_DTYPE = "datetime64[ns]"
+
+# Numpy converts datetime64 scalars to datetime.datetime only if microsecond or
+# smaller precision is used.
+#
+# TODO(https://github.com/googleapis/python-db-dtypes-pandas/issues/63): Keep
+# nanosecond precision when boxing scalars.
+_NP_BOX_DTYPE = "datetime64[us]"
 
 pandas_release = packaging.version.parse(pandas.__version__).release
 
@@ -149,12 +157,14 @@ def _box_func(self, x):
             return pandas.NaT
 
         try:
-            return x.astype("<M8[us]").astype(datetime.datetime).time()
+            return x.astype(_NP_BOX_DTYPE).item().time()
         except AttributeError:
-            x = numpy.datetime64(x)
-            return x.astype("<M8[us]").astype(datetime.datetime).time()
+            x = numpy.datetime64(
+                x, "ns"
+            )  # Integers are stored with nanosecond precision.
+            return x.astype(_NP_BOX_DTYPE).item().time()
 
-    __return_deltas = {"timedelta", "timedelta64", "timedelta64[ns]", "<m8", "<m8[ns]"}
+    __return_deltas = {"timedelta", "timedelta64", "timedelta64[ns]", "<m8", _NP_DTYPE}
 
     def astype(self, dtype, copy=True):
         deltas = self._ndarray - _NPEPOCH
@@ -253,10 +263,12 @@ def _box_func(self, x):
         if pandas.isna(x):
             return pandas.NaT
         try:
-            return x.astype("<M8[us]").astype(datetime.datetime).date()
+            return x.astype(_NP_BOX_DTYPE).item().date()
         except AttributeError:
-            x = numpy.datetime64(x)
-            return x.astype("<M8[us]").astype(datetime.datetime).date()
+            x = numpy.datetime64(
+                x, "ns"
+            )  # Integers are stored with nanosecond precision.
+            return x.astype(_NP_BOX_DTYPE).item().date()
 
     def astype(self, dtype, copy=True):
         stype = str(dtype)

db_dtypes/core.py

@@ -46,6 +46,12 @@ def construct_from_string(cls, name: str):
 class BaseDatetimeArray(
     pandas_backports.OpsMixin, pandas_backports.NDArrayBackedExtensionArray
 ):
+    # scalar used to denote NA value inside our self._ndarray, e.g. -1 for
+    # Categorical, iNaT for Period. Outside of object dtype, self.isna() should
+    # be exactly locations in self._ndarray with _internal_fill_value. See:
+    # https://github.com/pandas-dev/pandas/blob/main/pandas/core/arrays/_mixins.py
+    _internal_fill_value = numpy.datetime64("NaT")
+
     def __init__(self, values, dtype=None, copy: bool = False):
         if not (
             isinstance(values, numpy.ndarray) and values.dtype == numpy.dtype("<M8[ns]")

noxfile.py

@@ -19,6 +19,7 @@
 from __future__ import absolute_import
 import os
 import pathlib
+import re
 import shutil
 
 import nox
@@ -37,7 +38,9 @@
 nox.options.sessions = [
     "lint",
     "unit",
+    "unit_prerelease",
     "compliance",
+    "compliance_prerelease",
     "cover",
     "lint_setup_py",
     "blacken",
@@ -112,18 +115,106 @@ def default(session, tests_path):
     )
 
 
+def prerelease(session, tests_path):
+    constraints_path = str(
+        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+    )
+
+    # PyArrow prerelease packages are published to an alternative PyPI host.
+    # https://arrow.apache.org/docs/python/install.html#installing-nightly-packages
+    session.install(
+        "--extra-index-url",
+        "https://pypi.fury.io/arrow-nightlies/",
+        "--prefer-binary",
+        "--pre",
+        "--upgrade",
+        "pyarrow",
+    )
+    session.install(
+        "--extra-index-url",
+        "https://pypi.anaconda.org/scipy-wheels-nightly/simple",
+        "--prefer-binary",
+        "--pre",
+        "--upgrade",
+        "pandas",
+    )
+    session.install(
+        "mock",
+        "asyncmock",
+        "pytest",
+        "pytest-cov",
+        "pytest-asyncio",
+        "-c",
+        constraints_path,
+    )
+
+    # Because we test minimum dependency versions on the minimum Python
+    # version, the first version we test with in the unit tests sessions has a
+    # constraints file containing all dependencies and extras.
+    with open(
+        CURRENT_DIRECTORY
+        / "testing"
+        / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt",
+        encoding="utf-8",
+    ) as constraints_file:
+        constraints_text = constraints_file.read()
+
+    # Ignore leading whitespace and comment lines.
+    deps = [
+        match.group(1)
+        for match in re.finditer(
+            r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
+        )
+    ]
+
+    # We use --no-deps to ensure that pre-release versions aren't overwritten
+    # by the version ranges in setup.py.
+    session.install(*deps)
+    session.install("--no-deps", "-e", ".")
+
+    # Print out prerelease package versions.
+    session.run("python", "-m", "pip", "freeze")
+
+    # Run py.test against the unit tests.
+    session.run(
+        "py.test",
+        "--quiet",
+        f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml",
+        "--cov=db_dtypes",
+        "--cov=tests/unit",
+        "--cov-append",
+        "--cov-config=.coveragerc",
+        "--cov-report=",
+        "--cov-fail-under=0",
+        tests_path,
+        *session.posargs,
+    )
+
+
 @nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
 def compliance(session):
     """Run the compliance test suite."""
     default(session, os.path.join("tests", "compliance"))
 
 
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
+def compliance_prerelease(session):
+    """Run the compliance test suite with prerelease dependencies."""
+    prerelease(session, os.path.join("tests", "compliance"))
+
+
 @nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
 def unit(session):
     """Run the unit test suite."""
     default(session, os.path.join("tests", "unit"))
 
 
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
+def unit_prerelease(session):
+    """Run the unit test suite with prerelease dependencies."""
+    prerelease(session, os.path.join("tests", "unit"))
+
+
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
 def system(session):
     """Run the system test suite."""

owlbot.py

@@ -49,6 +49,10 @@
     ["noxfile.py"], r"[\"']google[\"']", '"db_dtypes"',
 )
 
+s.replace(
+    ["noxfile.py"], r"import shutil", "import re\nimport shutil",
+)
+
 s.replace(
     ["noxfile.py"], "--cov=google", "--cov=db_dtypes",
 )
@@ -64,7 +68,9 @@
 new_sessions = """
     "lint",
     "unit",
+    "unit_prerelease",
     "compliance",
+    "compliance_prerelease",
     "cover",
 """
 
@@ -83,17 +89,105 @@ def unit\(session\):
     """Run the unit test suite."""
     default\(session\)
 ''',
-    '''
+    r'''
+def prerelease(session, tests_path):
+    constraints_path = str(
+        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+    )
+
+    # PyArrow prerelease packages are published to an alternative PyPI host.
+    # https://arrow.apache.org/docs/python/install.html#installing-nightly-packages
+    session.install(
+        "--extra-index-url",
+        "https://pypi.fury.io/arrow-nightlies/",
+        "--prefer-binary",
+        "--pre",
+        "--upgrade",
+        "pyarrow",
+    )
+    session.install(
+        "--extra-index-url",
+        "https://pypi.anaconda.org/scipy-wheels-nightly/simple",
+        "--prefer-binary",
+        "--pre",
+        "--upgrade",
+        "pandas",
+    )
+    session.install(
+        "mock",
+        "asyncmock",
+        "pytest",
+        "pytest-cov",
+        "pytest-asyncio",
+        "-c",
+        constraints_path,
+    )
+
+    # Because we test minimum dependency versions on the minimum Python
+    # version, the first version we test with in the unit tests sessions has a
+    # constraints file containing all dependencies and extras.
+    with open(
+        CURRENT_DIRECTORY
+        / "testing"
+        / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt",
+        encoding="utf-8",
+    ) as constraints_file:
+        constraints_text = constraints_file.read()
+
+    # Ignore leading whitespace and comment lines.
+    deps = [
+        match.group(1)
+        for match in re.finditer(
+            r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE
+        )
+    ]
+
+    # We use --no-deps to ensure that pre-release versions aren't overwritten
+    # by the version ranges in setup.py.
+    session.install(*deps)
+    session.install("--no-deps", "-e", ".")
+
+    # Print out prerelease package versions.
+    session.run("python", "-m", "pip", "freeze")
+
+    # Run py.test against the unit tests.
+    session.run(
+        "py.test",
+        "--quiet",
+        f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml",
+        "--cov=db_dtypes",
+        "--cov=tests/unit",
+        "--cov-append",
+        "--cov-config=.coveragerc",
+        "--cov-report=",
+        "--cov-fail-under=0",
+        tests_path,
+        *session.posargs,
+    )
+
+
 @nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
 def compliance(session):
     """Run the compliance test suite."""
     default(session, os.path.join("tests", "compliance"))
 
 
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
+def compliance_prerelease(session):
+    """Run the compliance test suite with prerelease dependencies."""
+    prerelease(session, os.path.join("tests", "compliance"))
+
+
 @nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
 def unit(session):
     """Run the unit test suite."""
     default(session, os.path.join("tests", "unit"))
+
+
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
+def unit_prerelease(session):
+    """Run the unit test suite with prerelease dependencies."""
+    prerelease(session, os.path.join("tests", "unit"))
 ''',
 )