CI: run stubtest

docs/tests.md

@@ -11,5 +11,6 @@ Here are the most important options. Fore more details, please use `poe --help`.
   - Run only pytest: `poe pytest`
   - Run only pre-commit: `poe style`
 - Run tests against the installed stubs (this will install and uninstall the stubs): `poe test_dist`
+- Optional: Run stubtest to compare the installed pandas-stubs against pandas (this will fail): `poe stubtest`. If you have created an allowlist to ignore certain errors: `poe stubtest path_to_the_allow_list`
 
 These tests originally came from https://github.com/VirtusLab/pandas-stubs.

pyproject.toml

@@ -87,6 +87,10 @@ script = "scripts.test.run:pyright_src"
 help = "Run pyright on 'tests' using the installed stubs"
 script = "scripts.test:test(dist=True, type_checker='pyright')"
 
+[tool.poe.tasks.stubtest]
+script = "scripts.test:stubtest(allowlist)"
+help = "Run stubtest to compare the installed stubs against pandas"
+args = [{ name = "allowlist", positional = true, default = "", required = false, help= "Path to an allowlist (optional)" }]
 
 
 [tool.black]

scripts/_job.py

@@ -63,5 +63,8 @@ def run_job(steps: List[Step]) -> None:
         end = time.perf_counter()
         logger.success(f"End: '{step.name}', runtime: {end - start:.3f} seconds.")
 
+    if not failed:
+        __rollback_job(rollback_steps)
+
     if failed:
         sys.exit(1)

scripts/test/__init__.py

@@ -1,3 +1,5 @@
+from copy import copy
+from functools import partial
 from typing import Literal
 
 from scripts._job import run_job
@@ -10,8 +12,6 @@
     _step.rename_src,
     _step.mypy_dist,
     _step.pyright_dist,
-    _step.uninstall_dist,
-    _step.restore_src,
 ]
 
 
@@ -33,3 +33,9 @@ def test(
         steps = [step for step in steps if remove not in step.name]
 
     run_job(steps)
+
+
+def stubtest(allowlist: str):
+    stubtest = copy(_step.stubtest)
+    stubtest.run = partial(_step.stubtest.run, allowlist=allowlist)
+    run_job(_DIST_STEPS[:-2] + [stubtest])

scripts/test/_step.py

@@ -26,5 +26,6 @@
 pyright_dist = Step(
     name="Run pyright on 'tests' using the installed stubs", run=run.pyright_dist
 )
-uninstall_dist = Step(name="Uninstall pandas-stubs", run=run.uninstall_dist)
-restore_src = Step(name="Restore local stubs", run=run.restore_src)
+stubtest = Step(
+    name="Run stubtest to compare the installed stubs against pandas", run=run.stubtest
+)

scripts/test/run.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 import subprocess
+import sys
 
 
 def mypy_src():
@@ -22,6 +23,22 @@ def style():
     subprocess.run(cmd, check=True)
 
 
+def stubtest(allowlist: str = ""):
+    cmd = [
+        sys.executable,
+        "-m",
+        "mypy.stubtest",
+        "pandas",
+        "--concise",
+        "--ignore-missing-stub",
+        "--mypy-config-file",
+        "pyproject.toml",
+    ]
+    if allowlist:
+        cmd += ["--allowlist", allowlist]
+    subprocess.run(cmd, check=True)
+
+
 def build_dist():
     cmd = ["poetry", "build", "-f", "wheel"]
     subprocess.run(cmd, check=True)