Merge pull request #4105 from tybug/conjecture-typing

Add more internal type hints

Author: tybug

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch adds more type hints to internal Hypothesis code.

hypothesis-python/src/hypothesis/extra/numpy.py

@@ -531,7 +531,7 @@ def arrays(
             lambda s: arrays(dtype, s, elements=elements, fill=fill, unique=unique)
         )
     # From here on, we're only dealing with values and it's relatively simple.
-    dtype = np.dtype(dtype)  # type: ignore[arg-type,assignment]
+    dtype = np.dtype(dtype)  # type: ignore[arg-type]
     assert isinstance(dtype, np.dtype)  # help mypy out a bit...
     if elements is None or isinstance(elements, Mapping):
         if dtype.kind in ("m", "M") and "[" not in dtype.str:

hypothesis-python/src/hypothesis/internal/charmap.py

@@ -15,18 +15,64 @@
 import sys
 import tempfile
 import unicodedata
+from collections.abc import Iterable
 from functools import lru_cache
+from pathlib import Path
+from typing import TYPE_CHECKING, Literal, Optional
 
 from hypothesis.configuration import storage_directory
 from hypothesis.control import _current_build_context
 from hypothesis.errors import InvalidArgument
-from hypothesis.internal.intervalsets import IntervalSet
-
-intervals = tuple[tuple[int, int], ...]
-cache_type = dict[tuple[tuple[str, ...], int, int, intervals], IntervalSet]
-
-
-def charmap_file(fname="charmap"):
+from hypothesis.internal.intervalsets import IntervalSet, IntervalsT
+
+if TYPE_CHECKING:
+    from typing import TypeAlias
+
+# See https://en.wikipedia.org/wiki/Unicode_character_property#General_Category
+CategoryName: "TypeAlias" = Literal[
+    "L",  #  Letter
+    "Lu",  # Letter, uppercase
+    "Ll",  # Letter, lowercase
+    "Lt",  # Letter, titlecase
+    "Lm",  # Letter, modifier
+    "Lo",  # Letter, other
+    "M",  #  Mark
+    "Mn",  # Mark, nonspacing
+    "Mc",  # Mark, spacing combining
+    "Me",  # Mark, enclosing
+    "N",  #  Number
+    "Nd",  # Number, decimal digit
+    "Nl",  # Number, letter
+    "No",  # Number, other
+    "P",  #  Punctuation
+    "Pc",  # Punctuation, connector
+    "Pd",  # Punctuation, dash
+    "Ps",  # Punctuation, open
+    "Pe",  # Punctuation, close
+    "Pi",  # Punctuation, initial quote
+    "Pf",  # Punctuation, final quote
+    "Po",  # Punctuation, other
+    "S",  #  Symbol
+    "Sm",  # Symbol, math
+    "Sc",  # Symbol, currency
+    "Sk",  # Symbol, modifier
+    "So",  # Symbol, other
+    "Z",  #  Separator
+    "Zs",  # Separator, space
+    "Zl",  # Separator, line
+    "Zp",  # Separator, paragraph
+    "C",  #  Other
+    "Cc",  # Other, control
+    "Cf",  # Other, format
+    "Cs",  # Other, surrogate
+    "Co",  # Other, private use
+    "Cn",  # Other, not assigned
+]
+Categories: "TypeAlias" = Iterable[CategoryName]
+CategoriesTuple: "TypeAlias" = tuple[CategoryName, ...]
+
+
+def charmap_file(fname: str = "charmap") -> Path:
     return storage_directory(
         "unicode_data", unicodedata.unidata_version, f"{fname}.json.gz"
     )
@@ -35,7 +81,7 @@ def charmap_file(fname="charmap"):
 _charmap = None
 
 
-def charmap():
+def charmap() -> dict[CategoryName, IntervalsT]:
     """Return a dict that maps a Unicode category, to a tuple of 2-tuples
     covering the codepoint intervals for characters in that category.
 
@@ -49,8 +95,8 @@ def charmap():
     if _charmap is None:
         f = charmap_file()
         try:
-            with gzip.GzipFile(f, "rb") as i:
-                tmp_charmap = dict(json.load(i))
+            with gzip.GzipFile(f, "rb") as d:
+                tmp_charmap = dict(json.load(d))
 
         except Exception:
             # This loop is reduced to using only local variables for performance;
@@ -63,9 +109,9 @@ def charmap():
             for i in range(1, sys.maxunicode + 1):
                 cat = category(chr(i))
                 if cat != last_cat:
-                    tmp_charmap.setdefault(last_cat, []).append([last_start, i - 1])
+                    tmp_charmap.setdefault(last_cat, []).append((last_start, i - 1))
                     last_cat, last_start = cat, i
-            tmp_charmap.setdefault(last_cat, []).append([last_start, sys.maxunicode])
+            tmp_charmap.setdefault(last_cat, []).append((last_start, sys.maxunicode))
 
             try:
                 # Write the Unicode table atomically
@@ -135,10 +181,10 @@ def intervals_from_codec(codec_name: str) -> IntervalSet:  # pragma: no cover
     return res
 
 
-_categories = None
+_categories: Optional[Categories] = None
 
 
-def categories():
+def categories() -> Categories:
     """Return a tuple of Unicode categories in a normalised order.
 
     >>> categories() # doctest: +ELLIPSIS
@@ -147,15 +193,16 @@ def categories():
     global _categories
     if _categories is None:
         cm = charmap()
-        _categories = sorted(cm.keys(), key=lambda c: len(cm[c]))
-        _categories.remove("Cc")  # Other, Control
-        _categories.remove("Cs")  # Other, Surrogate
-        _categories.append("Cc")
-        _categories.append("Cs")
-    return tuple(_categories)
+        categories = sorted(cm.keys(), key=lambda c: len(cm[c]))
+        categories.remove("Cc")  # Other, Control
+        categories.remove("Cs")  # Other, Surrogate
+        categories.append("Cc")
+        categories.append("Cs")
+        _categories = tuple(categories)
+    return _categories
 
 
-def as_general_categories(cats, name="cats"):
+def as_general_categories(cats: Categories, name: str = "cats") -> CategoriesTuple:
     """Return a tuple of Unicode categories in a normalised order.
 
     This function expands one-letter designations of a major class to include
@@ -170,8 +217,6 @@ def as_general_categories(cats, name="cats"):
     If the collection ``cats`` includes any elements that do not represent a
     major class or a class with subclass, a deprecation warning is raised.
     """
-    if cats is None:
-        return None
     major_classes = ("L", "M", "N", "P", "S", "Z", "C")
     cs = categories()
     out = set(cats)
@@ -186,10 +231,10 @@ def as_general_categories(cats, name="cats"):
     return tuple(c for c in cs if c in out)
 
 
-category_index_cache = {(): ()}
+category_index_cache: dict[frozenset[CategoryName], IntervalsT] = {frozenset(): ()}
 
 
-def _category_key(cats):
+def _category_key(cats: Optional[Iterable[str]]) -> CategoriesTuple:
     """Return a normalised tuple of all Unicode categories that are in
     `include`, but not in `exclude`.
 
@@ -205,7 +250,7 @@ def _category_key(cats):
     return tuple(c for c in cs if c in cats)
 
 
-def _query_for_key(key):
+def _query_for_key(key: Categories) -> IntervalsT:
     """Return a tuple of codepoint intervals covering characters that match one
     or more categories in the tuple of categories `key`.
 
@@ -214,10 +259,13 @@ def _query_for_key(key):
     >>> _query_for_key(('Zl', 'Zp', 'Co'))
     ((8232, 8233), (57344, 63743), (983040, 1048573), (1048576, 1114109))
     """
+    key = tuple(key)
+    # ignore ordering on the cache key to increase potential cache hits.
+    cache_key = frozenset(key)
     context = _current_build_context.value
     if context is None or not context.data.provider.avoid_realization:
         try:
-            return category_index_cache[key]
+            return category_index_cache[cache_key]
         except KeyError:
             pass
     elif not key:  # pragma: no cover  # only on alternative backends
@@ -231,21 +279,23 @@ def _query_for_key(key):
         )
     assert isinstance(result, IntervalSet)
     if context is None or not context.data.provider.avoid_realization:
-        category_index_cache[key] = result.intervals
+        category_index_cache[cache_key] = result.intervals
     return result.intervals
 
 
-limited_category_index_cache: cache_type = {}
+limited_category_index_cache: dict[
+    tuple[CategoriesTuple, int, int, IntervalsT, IntervalsT], IntervalSet
+] = {}
 
 
 def query(
     *,
-    categories=None,
-    min_codepoint=None,
-    max_codepoint=None,
-    include_characters="",
-    exclude_characters="",
-):
+    categories: Optional[Categories] = None,
+    min_codepoint: Optional[int] = None,
+    max_codepoint: Optional[int] = None,
+    include_characters: str = "",
+    exclude_characters: str = "",
+) -> IntervalSet:
     """Return a tuple of intervals covering the codepoints for all characters
     that meet the criteria.
 

hypothesis-python/src/hypothesis/internal/compat.py

@@ -36,6 +36,8 @@
         TypedDict as TypedDict,
         override as override,
     )
+
+    from hypothesis.internal.conjecture.engine import ConjectureRunner
 else:
     # In order to use NotRequired, we need the version of TypedDict included in Python 3.11+.
     if sys.version_info[:2] >= (3, 11):
@@ -129,7 +131,7 @@ def _hint_and_args(x):
     return (x, *get_args(x))
 
 
-def get_type_hints(thing):
+def get_type_hints(thing: object) -> dict[str, Any]:
     """Like the typing version, but tries harder and never errors.
 
     Tries harder: if the thing to inspect is a class but typing.get_type_hints
@@ -237,7 +239,7 @@ def extract_bits(x: int, /, width: Optional[int] = None) -> list[int]:
     bit_count = lambda self: sum(extract_bits(abs(self)))
 
 
-def bad_django_TestCase(runner):
+def bad_django_TestCase(runner: Optional["ConjectureRunner"]) -> bool:
     if runner is None or "django.test" not in sys.modules:
         return False
     else:  # pragma: no cover

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -569,7 +569,7 @@ def ir_ends(self) -> IntList:
 
     class _discarded(ExampleProperty):
         def begin(self) -> None:
-            self.result: "set[int]" = set()
+            self.result: set[int] = set()
 
         def finish(self) -> frozenset[int]:
             return frozenset(self.result)
@@ -583,7 +583,7 @@ def stop_example(self, i: int, *, discarded: bool) -> None:
     class _trivial(ExampleProperty):
         def begin(self) -> None:
             self.nontrivial = IntList.of_length(len(self.examples))
-            self.result: "set[int]" = set()
+            self.result: set[int] = set()
 
         def block(self, i: int) -> None:
             if not self.examples.blocks.trivial(i):

hypothesis-python/src/hypothesis/internal/conjecture/utils.py

@@ -160,9 +160,10 @@ def __init__(self, weights: Sequence[float], *, observe: bool = True):
             table[small.pop()][2] = zero
 
         self.table: "list[tuple[int, int, float]]" = []
-        for base, alternate, alternate_chance in table:  # type: ignore
+        for base, alternate, alternate_chance in table:
             assert isinstance(base, int)
             assert isinstance(alternate, int) or alternate is None
+            assert alternate_chance is not None
             if alternate is None:
                 self.table.append((base, base, alternate_chance))
             elif alternate < base:

hypothesis-python/src/hypothesis/internal/entropy.py

@@ -13,9 +13,9 @@
 import random
 import sys
 import warnings
-from collections.abc import Hashable
+from collections.abc import Generator, Hashable
 from itertools import count
-from typing import TYPE_CHECKING, Any, Callable
+from typing import TYPE_CHECKING, Any, Callable, Optional
 from weakref import WeakValueDictionary
 
 import hypothesis.core
@@ -28,9 +28,9 @@
     # we can't use this at runtime until from_type supports
     # protocols -- breaks ghostwriter tests
     class RandomLike(Protocol):
-        seed: Callable[..., Any]
-        getstate: Callable[[], Any]
-        setstate: Callable[..., Any]
+        def seed(self, *args: Any, **kwargs: Any) -> Any: ...
+        def getstate(self, *args: Any, **kwargs: Any) -> Any: ...
+        def setstate(self, *args: Any, **kwargs: Any) -> Any: ...
 
 else:  # pragma: no cover
     RandomLike = random.Random
@@ -39,11 +39,13 @@ class RandomLike(Protocol):
 # with their respective Random instances even as new ones are registered and old
 # ones go out of scope and get garbage collected.  Keys are ascending integers.
 _RKEY = count()
-RANDOMS_TO_MANAGE: WeakValueDictionary = WeakValueDictionary({next(_RKEY): random})
+RANDOMS_TO_MANAGE: WeakValueDictionary[int, RandomLike] = WeakValueDictionary(
+    {next(_RKEY): random}
+)
 
 
 class NumpyRandomWrapper:
-    def __init__(self):
+    def __init__(self) -> None:
         assert "numpy" in sys.modules
         # This class provides a shim that matches the numpy to stdlib random,
         # and lets us avoid importing Numpy until it's already in use.
@@ -54,7 +56,7 @@ def __init__(self):
         self.setstate = numpy.random.set_state
 
 
-NP_RANDOM = None
+NP_RANDOM: Optional[RandomLike] = None
 
 
 if not (PYPY or GRAALPY):
@@ -160,21 +162,21 @@ def get_seeder_and_restorer(
     """
     assert isinstance(seed, int)
     assert 0 <= seed < 2**32
-    states: dict = {}
+    states: dict[int, object] = {}
 
     if "numpy" in sys.modules:
         global NP_RANDOM
         if NP_RANDOM is None:
             # Protect this from garbage-collection by adding it to global scope
             NP_RANDOM = RANDOMS_TO_MANAGE[next(_RKEY)] = NumpyRandomWrapper()
 
-    def seed_all():
+    def seed_all() -> None:
         assert not states
         for k, r in RANDOMS_TO_MANAGE.items():
             states[k] = r.getstate()
             r.seed(seed)
 
-    def restore_all():
+    def restore_all() -> None:
         for k, state in states.items():
             r = RANDOMS_TO_MANAGE.get(k)
             if r is not None:  # i.e., hasn't been garbage-collected
@@ -185,7 +187,7 @@ def restore_all():
 
 
 @contextlib.contextmanager
-def deterministic_PRNG(seed=0):
+def deterministic_PRNG(seed: int = 0) -> Generator[None, None, None]:
     """Context manager that handles random.seed without polluting global state.
 
     See issue #1255 and PR #1295 for details and motivation - in short,

hypothesis-python/src/hypothesis/internal/filtering.py

@@ -282,7 +282,7 @@ def get_numeric_predicate_bounds(predicate: Predicate) -> ConstructivePredicate:
 
 
 def get_integer_predicate_bounds(predicate: Predicate) -> ConstructivePredicate:
-    kwargs, predicate = get_numeric_predicate_bounds(predicate)  # type: ignore
+    kwargs, predicate = get_numeric_predicate_bounds(predicate)
 
     if "min_value" in kwargs:
         if kwargs["min_value"] == -math.inf:
@@ -310,7 +310,7 @@ def get_integer_predicate_bounds(predicate: Predicate) -> ConstructivePredicate:
 
 
 def get_float_predicate_bounds(predicate: Predicate) -> ConstructivePredicate:
-    kwargs, predicate = get_numeric_predicate_bounds(predicate)  # type: ignore
+    kwargs, predicate = get_numeric_predicate_bounds(predicate)
 
     if "min_value" in kwargs:
         min_value = kwargs["min_value"]