Merge pull request #4126 from Zac-HD/various-cleanups

Various cleanups

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,4 @@
+RELEASE_TYPE: patch
+
+This patch tweaks the paths in ``@example(...)`` patches, so that
+both ``git apply`` and ``patch`` will work by default.

hypothesis-python/scripts/other-tests.sh

@@ -51,11 +51,6 @@ pip uninstall -y lark
 if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel == \'final\' and platform.python_implementation() not in ("PyPy", "GraalVM"))')" = "True" ] ; then
   pip install ".[codemods,cli]"
   $PYTEST tests/codemods/
-  pip install "$(grep -E 'black(==| @)' ../requirements/coverage.txt)"
-  if [ "$(python -c 'import sys; print(sys.version_info[:2] >= (3, 9))')" = "True" ] ; then
-    $PYTEST tests/patching/
-  fi
-  pip uninstall -y libcst
 
   if [ "$(python -c 'import sys; print(sys.version_info[:2] == (3, 8))')" = "True" ] ; then
     # Per NEP-29, this is the last version to support Python 3.8
@@ -64,6 +59,12 @@ if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel ==
     pip install "$(grep 'numpy==' ../requirements/coverage.txt)"
   fi
 
+  pip install "$(grep -E 'black(==| @)' ../requirements/coverage.txt)"
+  if [ "$(python -c 'import sys; print(sys.version_info[:2] >= (3, 9))')" = "True" ] ; then
+    $PYTEST tests/patching/
+  fi
+  pip uninstall -y libcst
+
   $PYTEST tests/ghostwriter/
   pip uninstall -y black
 

hypothesis-python/src/_hypothesis_ftz_detector.py

@@ -18,6 +18,14 @@
 
 import importlib
 import sys
+from typing import TYPE_CHECKING, Callable, Optional, Set, Tuple
+
+if TYPE_CHECKING:
+    from multiprocessing import Queue
+    from typing import TypeAlias
+
+FTZCulprits: "TypeAlias" = Tuple[Optional[bool], Set[str]]
+
 
 KNOWN_EVER_CULPRITS = (
     # https://moyix.blogspot.com/2022/09/someones-been-messing-with-my-subnormals.html
@@ -35,32 +43,34 @@
 )
 
 
-def flush_to_zero():
+def flush_to_zero() -> bool:
     # If this subnormal number compares equal to zero we have a problem
     return 2.0**-1073 == 0
 
 
-def run_in_process(fn, *args):
+def run_in_process(fn: Callable[..., FTZCulprits], *args: object) -> FTZCulprits:
     import multiprocessing as mp
 
     mp.set_start_method("spawn", force=True)
-    q = mp.Queue()
+    q: "Queue[FTZCulprits]" = mp.Queue()
     p = mp.Process(target=target, args=(q, fn, *args))
     p.start()
     retval = q.get()
     p.join()
     return retval
 
 
-def target(q, fn, *args):
+def target(
+    q: "Queue[FTZCulprits]", fn: Callable[..., FTZCulprits], *args: object
+) -> None:
     q.put(fn(*args))
 
 
-def always_imported_modules():
+def always_imported_modules() -> FTZCulprits:
     return flush_to_zero(), set(sys.modules)
 
 
-def modules_imported_by(mod):
+def modules_imported_by(mod: str) -> FTZCulprits:
     """Return the set of modules imported transitively by mod."""
     before = set(sys.modules)
     try:
@@ -77,7 +87,7 @@ def modules_imported_by(mod):
 CHECKED_CACHE = set()
 
 
-def identify_ftz_culprits():
+def identify_ftz_culprits() -> str:
     """Find the modules in sys.modules which cause "mod" to be imported."""
     # If we've run this function before, return the same result.
     global KNOWN_FTZ
@@ -94,7 +104,7 @@ def identify_ftz_culprits():
     # that importing them in a new process sets the FTZ state.  As a heuristic, we'll
     # start with packages known to have ever enabled FTZ, then top-level packages as
     # a way to eliminate large fractions of the search space relatively quickly.
-    def key(name):
+    def key(name: str) -> Tuple[bool, int, str]:
         """Prefer known-FTZ modules, then top-level packages, then alphabetical."""
         return (name not in KNOWN_EVER_CULPRITS, name.count("."), name)
 

hypothesis-python/src/hypothesis/configuration.py

@@ -12,22 +12,22 @@
 import sys
 import warnings
 from pathlib import Path
+from typing import Union
 
 import _hypothesis_globals
 
 from hypothesis.errors import HypothesisSideeffectWarning
 
 __hypothesis_home_directory_default = Path.cwd() / ".hypothesis"
-
 __hypothesis_home_directory = None
 
 
-def set_hypothesis_home_dir(directory):
+def set_hypothesis_home_dir(directory: Union[str, Path, None]) -> None:
     global __hypothesis_home_directory
     __hypothesis_home_directory = None if directory is None else Path(directory)
 
 
-def storage_directory(*names, intent_to_write=True):
+def storage_directory(*names: str, intent_to_write: bool = True) -> Path:
     if intent_to_write:
         check_sideeffect_during_initialization(
             "accessing storage for {}", "/".join(names)

hypothesis-python/src/hypothesis/errors.py

@@ -214,7 +214,7 @@ class StopTest(BaseException):
     the Hypothesis engine, which should then continue normally.
     """
 
-    def __init__(self, testcounter):
+    def __init__(self, testcounter: int) -> None:
         super().__init__(repr(testcounter))
         self.testcounter = testcounter
 
@@ -230,7 +230,7 @@ class Found(HypothesisException):
 class RewindRecursive(Exception):
     """Signal that the type inference should be rewound due to recursive types. Internal use only."""
 
-    def __init__(self, target):
+    def __init__(self, target: object) -> None:
         self.target = target
 
 

hypothesis-python/src/hypothesis/extra/_patching.py

@@ -18,11 +18,13 @@
 during fuzzing.
 """
 
+import ast
 import difflib
 import hashlib
 import inspect
 import re
 import sys
+import types
 from ast import literal_eval
 from contextlib import suppress
 from datetime import date, datetime, timedelta, timezone
@@ -31,6 +33,7 @@
 import libcst as cst
 from libcst import matchers as m
 from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand
+from libcst.metadata import ExpressionContext, ExpressionContextProvider
 
 from hypothesis.configuration import storage_directory
 from hypothesis.version import __version__
@@ -148,18 +151,64 @@ def get_patch_for(func, failing_examples, *, strip_via=()):
     except Exception:
         return None
 
+    modules_in_test_scope = sorted(
+        (
+            (k, v)
+            for (k, v) in module.__dict__.items()
+            if isinstance(v, types.ModuleType)
+        ),
+        key=lambda kv: len(kv[1].__name__),
+    )
+
     # The printed examples might include object reprs which are invalid syntax,
     # so we parse here and skip over those.  If _none_ are valid, there's no patch.
     call_nodes = []
     for ex, via in set(failing_examples):
         with suppress(Exception):
-            node = cst.parse_expression(ex)
-            assert isinstance(node, cst.Call), node
+            node = cst.parse_module(ex)
+            the_call = node.body[0].body[0].value
+            assert isinstance(the_call, cst.Call), the_call
             # Check for st.data(), which doesn't support explicit examples
             data = m.Arg(m.Call(m.Name("data"), args=[m.Arg(m.Ellipsis())]))
-            if m.matches(node, m.Call(args=[m.ZeroOrMore(), data, m.ZeroOrMore()])):
+            if m.matches(the_call, m.Call(args=[m.ZeroOrMore(), data, m.ZeroOrMore()])):
                 return None
+
+            # Many reprs use the unqualified name of the type, e.g. np.array()
+            # -> array([...]), so here we find undefined names and look them up
+            # on each module which was in the test's global scope.
+            names = {}
+            for anode in ast.walk(ast.parse(ex, "eval")):
+                if (
+                    isinstance(anode, ast.Name)
+                    and isinstance(anode.ctx, ast.Load)
+                    and anode.id not in names
+                    and anode.id not in module.__dict__
+                ):
+                    for k, v in modules_in_test_scope:
+                        if anode.id in v.__dict__:
+                            names[anode.id] = cst.parse_expression(f"{k}.{anode.id}")
+                            break
+
+            # LibCST doesn't track Load()/Store() state of names by default, so we have
+            # to do a bit of a dance here, *and* explicitly handle keyword arguments
+            # which are treated as Load() context - but even if that's fixed later
+            # we'll still want to support older versions.
+            with suppress(Exception):
+                wrapper = cst.metadata.MetadataWrapper(node)
+                kwarg_names = {
+                    a.keyword for a in m.findall(wrapper, m.Arg(keyword=m.Name()))
+                }
+                node = m.replace(
+                    wrapper,
+                    m.Name(value=m.MatchIfTrue(names.__contains__))
+                    & m.MatchMetadata(ExpressionContextProvider, ExpressionContext.LOAD)
+                    & m.MatchIfTrue(lambda n, k=kwarg_names: n not in k),
+                    replacement=lambda node, _, ns=names: ns[node.value],
+                )
+            node = node.body[0].body[0].value
+            assert isinstance(node, cst.Call), node
             call_nodes.append((node, via))
+
     if not call_nodes:
         return None
 
@@ -205,8 +254,8 @@ def make_patch(triples, *, msg="Hypothesis: add explicit examples", when=None):
         ud = difflib.unified_diff(
             source_before.splitlines(keepends=True),
             source_after.splitlines(keepends=True),
-            fromfile=str(fname),
-            tofile=str(fname),
+            fromfile=f"./{fname}",  # git strips the first part of the path by default
+            tofile=f"./{fname}",
         )
         diffs.append("".join(ud))
     return "".join(diffs)

hypothesis-python/src/hypothesis/internal/cache.py

@@ -307,7 +307,7 @@ class LRUCache:
     # Anecdotally, OrderedDict seems quite competitive with lru_cache, but perhaps
     # that is localized to our access patterns.
 
-    def __init__(self, max_size):
+    def __init__(self, max_size: int) -> None:
         assert max_size > 0
         self.max_size = max_size
         self._threadlocal = threading.local()

hypothesis-python/src/hypothesis/internal/cathetus.py

@@ -12,7 +12,7 @@
 from sys import float_info
 
 
-def cathetus(h, a):
+def cathetus(h: float, a: float) -> float:
     """Given the lengths of the hypotenuse and a side of a right triangle,
     return the length of the other side.
 

hypothesis-python/src/hypothesis/internal/compat.py

@@ -110,7 +110,7 @@ def int_to_byte(i: int) -> bytes:
     return bytes([i])
 
 
-def is_typed_named_tuple(cls):
+def is_typed_named_tuple(cls: type) -> bool:
     """Return True if cls is probably a subtype of `typing.NamedTuple`.
 
     Unfortunately types created with `class T(NamedTuple):` actually