Merge pull request #4342 from tybug/scrutineer-sort

Sort scrutineer reported lines

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+When reporting the always-failing, never-passing lines from the |Phase.explain| phase, we now sort the reported lines so that local code shows up first, then third-party library code, then standard library code.

hypothesis-python/src/hypothesis/internal/scrutineer.py

@@ -13,6 +13,7 @@
 import re
 import subprocess
 import sys
+import sysconfig
 import types
 from collections import defaultdict
 from collections.abc import Iterable
@@ -27,12 +28,10 @@
 
 if TYPE_CHECKING:
     from typing import TypeAlias
-else:
-    TypeAlias = object
 
-Location: TypeAlias = tuple[str, int]
-Branch: TypeAlias = tuple[Optional[Location], Location]
-Trace: TypeAlias = set[Branch]
+Location: "TypeAlias" = tuple[str, int]
+Branch: "TypeAlias" = tuple[Optional[Location], Location]
+Trace: "TypeAlias" = set[Branch]
 
 
 @lru_cache(maxsize=None)
@@ -215,18 +214,41 @@ def get_explaining_locations(traces):
     }
 
 
-LIB_DIR = str(Path(sys.executable).parent / "lib")
+# see e.g. https://docs.python.org/3/library/sysconfig.html#posix-user
+# for examples of these path schemes
+STDLIB_DIRS = [
+    Path(sysconfig.get_path("platstdlib")).resolve(),
+    Path(sysconfig.get_path("stdlib")).resolve(),
+]
+SITE_PACKAGES_DIRS = [
+    Path(sysconfig.get_path("purelib")).resolve(),
+    Path(sysconfig.get_path("platlib")).resolve(),
+]
+
 EXPLANATION_STUB = (
     "Explanation:",
     "    These lines were always and only run by failing examples:",
 )
 
 
-def make_report(explanations, cap_lines_at=5):
+# show local files first, then site-packages, then stdlib
+def _sort_key(path, lineno):
+    path = Path(path).resolve()
+    # site-packages may be a subdir of stdlib or platlib, so it's important to
+    # check is_relative_to for this before the stdlib.
+    if any(path.is_relative_to(p) for p in SITE_PACKAGES_DIRS):
+        return (1, path, lineno)
+    if any(path.is_relative_to(p) for p in STDLIB_DIRS):
+        return (2, path, lineno)
+    return (0, path, lineno)
+
+
+def make_report(explanations, *, cap_lines_at=5):
     report = defaultdict(list)
     for origin, locations in explanations.items():
+        locations = list(locations)
+        locations.sort(key=lambda v: _sort_key(v[0], v[1]))
         report_lines = [f"        {fname}:{lineno}" for fname, lineno in locations]
-        report_lines.sort(key=lambda line: (line.startswith(LIB_DIR), line))
         if len(report_lines) > cap_lines_at + 1:
             msg = "        (and {} more with settings.verbosity >= verbose)"
             report_lines[cap_lines_at:] = [msg.format(len(report_lines[cap_lines_at:]))]

hypothesis-python/tests/ghostwriter/try-writing-for-installed.py

@@ -19,10 +19,10 @@
 just have such weird semantics that we don't _want_ to support them.
 """
 
-import distutils.sysconfig as sysconfig
 import multiprocessing
 import os
 import subprocess
+import sysconfig
 
 skip = (
     "idlelib curses antigravity pip prompt_toolkit IPython .popen_ django. .test. "

hypothesis-python/tests/nocover/test_scrutineer.py

@@ -8,12 +8,16 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import json
 import sys
+import sysconfig
 
 import pytest
 
+from hypothesis import given, note, settings, strategies as st
 from hypothesis.internal.compat import PYPY
 from hypothesis.internal.scrutineer import make_report
+from hypothesis.vendor import pretty
 
 # We skip tracing for explanations under PyPy, where it has a large performance
 # impact, or if there is already a trace function (e.g. coverage or a debugger)
@@ -105,3 +109,39 @@ def test(x):
 def test_skips_uninformative_locations(testdir):
     pytest_stdout, _ = get_reports(NO_SHOW_CONTEXTLIB, testdir=testdir)
     assert "Explanation:" not in pytest_stdout
+
+
+@given(st.randoms())
+@settings(max_examples=5)
+def test_report_sort(random):
+    # show local files first, then site-packages, then stdlib
+
+    lines = [
+        # local
+        (__file__, 10),
+        # site-packages
+        (pytest.__file__, 123),
+        (pytest.__file__, 124),
+        # stdlib
+        (json.__file__, 43),
+        (json.__file__, 42),
+    ]
+    random.shuffle(lines)
+    explanations = {"origin": lines}
+    report = make_report(explanations)
+    report_lines = report["origin"][2:]
+    report_lines = [line.strip() for line in report_lines]
+
+    expected_lines = [
+        f"{__file__}:10",
+        f"{pytest.__file__}:123",
+        f"{pytest.__file__}:124",
+        f"{json.__file__}:42",
+        f"{json.__file__}:43",
+    ]
+
+    note(f"sysconfig.get_paths(): {pretty.pretty(sysconfig.get_paths())}")
+    note(f"actual lines: {pretty.pretty(report_lines)}")
+    note(f"expected lines: {pretty.pretty(expected_lines)}")
+
+    assert report_lines == expected_lines