Merge pull request #4348 from tybug/collect-ast-constants

Initial work on collecting ast constants

Author: Zac-HD

.github/workflows/main.yml

@@ -252,12 +252,14 @@ jobs:
           version: ${{ env.EMSCRIPTEN_VERSION }}
       - name: Build
         run: |
+          # TODO remove https://github.com/pyodide/pyodide/issues/5585
+          pip install -U wheel==0.45.1
           pip install pyodide-build==$PYODIDE_VERSION
           cd hypothesis-python/
           CFLAGS=-g2 LDFLAGS=-g2 pyodide build
       - name: Set up Pyodide venv and install dependencies
         run: |
-          pip install --upgrade setuptools pip wheel build
+          pip install --upgrade setuptools pip wheel==0.45.1 build
           python -m build --wheel hypothesis-python --outdir dist/
           pip download --dest=dist/ hypothesis-python/ pytest tzdata  # fetch all the wheels
 

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+Lays some groundwork for future work on collecting interesting literals from the code being tested, for increased bug-finding power (:issue:`3127`). There is no user-visible change (yet!)

hypothesis-python/src/hypothesis/internal/constants_ast.py

@@ -0,0 +1,136 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Copyright the Hypothesis Authors.
+# Individual contributors are listed in AUTHORS.rst and the git log.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+
+import ast
+import inspect
+import math
+import sys
+from ast import AST, Constant, Expr, NodeVisitor, UnaryOp, USub
+from functools import lru_cache
+from types import ModuleType
+from typing import TYPE_CHECKING, Optional, Union
+
+from hypothesis.internal.escalation import is_hypothesis_file
+from hypothesis.internal.scrutineer import ModuleLocation
+
+if TYPE_CHECKING:
+    from typing import TypeAlias
+
+ConstantT: "TypeAlias" = Union[int, float, bool, bytes, str]
+
+
+class ConstantVisitor(NodeVisitor):
+    def __init__(self):
+        super().__init__()
+        self.constants: set[ConstantT] = set()
+
+    def _add_constant(self, constant: object) -> None:
+        self.constants |= self._unfold_constant(constant)
+
+    def visit_UnaryOp(self, node: UnaryOp) -> None:
+        # `a = -1` is actually a combination of a USub and the constant 1.
+        if (
+            isinstance(node.op, USub)
+            and isinstance(node.operand, Constant)
+            and isinstance(node.operand.value, (int, float))
+            and not isinstance(node.operand.value, bool)
+        ):
+            self._add_constant(-node.operand.value)
+            # don't recurse on this node to avoid adding the positive variant
+            return
+
+        self.generic_visit(node)
+
+    def visit_Expr(self, node: Expr) -> None:
+        if isinstance(node.value, Constant) and isinstance(node.value.value, str):
+            return
+
+        self.generic_visit(node)
+
+    def visit_JoinedStr(self, node):
+        # dont recurse on JoinedStr, i.e. f strings. Constants that appear *only*
+        # in f strings are unlikely to be helpful.
+        return
+
+    @classmethod
+    def _unfold_constant(cls, value: object) -> set[ConstantT]:
+        if isinstance(value, str) and (
+            len(value) > 20 or value.isspace() or value == ""
+        ):
+            # discard long strings, which are unlikely to be useful.
+            return set()
+        if isinstance(value, bool):
+            return set()
+        if isinstance(value, float) and math.isinf(value):
+            # we already upweight inf.
+            return set()
+        if isinstance(value, (int, float, bytes, str)):
+            return {value}
+        # I don't kow what case could go here, but am also not confident there
+        # isn't one.
+        return set()  # pragma: no cover
+
+    def visit_Constant(self, node):
+        self._add_constant(node.value)
+        self.generic_visit(node)
+
+
+@lru_cache(1024)
+def constants_from_ast(tree: AST) -> set[ConstantT]:
+    visitor = ConstantVisitor()
+    visitor.visit(tree)
+    return visitor.constants
+
+
+@lru_cache(1024)
+def _module_ast(module: ModuleType) -> Optional[AST]:
+    try:
+        source = inspect.getsource(module)
+        tree = ast.parse(source)
+    except Exception:
+        return None
+
+    return tree
+
+
+def local_modules() -> tuple[ModuleType, ...]:
+    modules = []
+    for module in sys.modules.values():
+        if not hasattr(module, "__file__"):
+            continue
+        if module.__file__ is None:  # pragma: no cover
+            continue
+
+        if ModuleLocation.from_path(module.__file__) is not ModuleLocation.LOCAL:
+            continue
+
+        modules.append(module)
+    return tuple(modules)
+
+
+def local_constants():
+    constants = set()
+    for module in local_modules():
+        # normally, hypothesis is a third-party library and is not returned
+        # by local_modules. However, if it is installed as an editable package
+        # with pip install -e, then we will pick up on it. Just hardcode an
+        # ignore here.
+
+        # this is actually covered by test_constants_from_running_file, but
+        # not in the same process.
+        if is_hypothesis_file(module.__file__):  # pragma: no cover
+            continue
+
+        tree = _module_ast(module)
+        if tree is None:  # pragma: no cover
+            continue
+        constants |= constants_from_ast(tree)
+
+    return constants

hypothesis-python/src/hypothesis/internal/scrutineer.py

@@ -17,6 +17,7 @@
 import types
 from collections import defaultdict
 from collections.abc import Iterable
+from enum import IntEnum
 from functools import lru_cache, reduce
 from os import sep
 from pathlib import Path
@@ -231,16 +232,27 @@ def get_explaining_locations(traces):
 )
 
 
+class ModuleLocation(IntEnum):
+    LOCAL = 0
+    SITE_PACKAGES = 1
+    STDLIB = 2
+
+    @classmethod
+    @lru_cache(1024)
+    def from_path(cls, path: str) -> "ModuleLocation":
+        path = Path(path).resolve()
+        # site-packages may be a subdir of stdlib or platlib, so it's important to
+        # check is_relative_to for this before the stdlib.
+        if any(path.is_relative_to(p) for p in SITE_PACKAGES_DIRS):
+            return cls.SITE_PACKAGES
+        if any(path.is_relative_to(p) for p in STDLIB_DIRS):
+            return cls.STDLIB
+        return cls.LOCAL
+
+
 # show local files first, then site-packages, then stdlib
-def _sort_key(path, lineno):
-    path = Path(path).resolve()
-    # site-packages may be a subdir of stdlib or platlib, so it's important to
-    # check is_relative_to for this before the stdlib.
-    if any(path.is_relative_to(p) for p in SITE_PACKAGES_DIRS):
-        return (1, path, lineno)
-    if any(path.is_relative_to(p) for p in STDLIB_DIRS):
-        return (2, path, lineno)
-    return (0, path, lineno)
+def _sort_key(path: str, lineno: int) -> tuple[int, str, int]:
+    return (ModuleLocation.from_path(path), path, lineno)
 
 
 def make_report(explanations, *, cap_lines_at=5):

hypothesis-python/tests/cover/test_constants_ast.py

@@ -0,0 +1,156 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Copyright the Hypothesis Authors.
+# Individual contributors are listed in AUTHORS.rst and the git log.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+
+import ast
+import subprocess
+import sys
+import textwrap
+from types import ModuleType
+
+import pytest
+
+from hypothesis import given, strategies as st
+from hypothesis.internal.constants_ast import _module_ast, constants_from_ast
+
+from tests.common.utils import skipif_emscripten
+
+
+@pytest.mark.parametrize(
+    "source, expected",
+    [
+        (
+            """
+            a1 = 42
+            a2 = 3.14
+            a3 = 'test1'
+            a4 = b'test2'
+            a5 = (1, 2)
+            a6 = frozenset([3])
+            """,
+            {42, 3.14, "test1", b"test2", 1, 2, 3},
+        ),
+        ("a = (1, (2, 3), frozenset([4, 5]))", {1, 2, 3, 4, 5}),
+        ("a = {'b': 1}", {"b", 1}),
+        ("a = [1]", {1}),
+        ("a = +42", {42}),
+        ("a = 1 + 2", {1, 2}),
+        ("a = ~ 42", {42}),
+        # the following cases are ignored:
+        # * booleans
+        # * math.inf and math.nan (not constants, but we don't want to collect them
+        #   even if they were)
+        # * f-strings
+        # * long strings
+        # * pure-whitespace strings
+        # * standalone string expressions (strings not assigned to a variable).
+        #   This covers docstrings of all kinds.
+        ("a = True", set()),
+        ("a = False", set()),
+        ("a = not False", set()),
+        ("a = 1e999", set()),
+        ("a = math.inf", set()),
+        ("a = math.nan", set()),
+        ('a = f"test {x}"', set()),
+        (f'a = "{"b" * 100}"', set()),
+        ('a = ""', set()),
+        ('a = " "', set()),
+        ('a = "\\n    \\n  \\n"', set()),
+        ("'test'", set()),
+        ("'test with \\n newlines'", set()),
+    ],
+)
+def test_constants_from_ast(source, expected):
+    source = textwrap.dedent(source)
+    tree = ast.parse(source)
+    assert constants_from_ast(tree) == expected
+
+
+@given(st.integers(max_value=-1))
+def test_parses_negatives(n):
+    assert constants_from_ast(ast.parse(f"a = {n}")) == {n}
+
+
+constants = st.one_of(
+    st.integers(),
+    st.floats(allow_nan=False, allow_infinity=False),
+    st.binary(),
+    # constants_from_ast ignores the following strings:
+    # * empty strings
+    # * long strings
+    # * strings which are entirely spaces
+    st.text(min_size=1, max_size=10).filter(lambda s: not s.isspace()),
+)
+
+
+@given(st.tuples(constants))
+def test_tuple_constants(value):
+    tree = ast.parse(str(value))
+    assert set(constants_from_ast(tree)) == set(value)
+
+
+@given(st.frozensets(constants))
+def test_frozenset_constants(value):
+    tree = ast.parse(str(value))
+    assert set(constants_from_ast(tree)) == set(value)
+
+
+@skipif_emscripten
+def test_constants_from_running_file(tmp_path):
+    p = tmp_path / "test_constants.py"
+    p.write_text(
+        textwrap.dedent(
+            """
+        import sys
+        # stdlib
+        import json
+        # third-party
+        import pytest
+        import hypothesis
+        from hypothesis.internal.constants_ast import local_constants
+
+        # these modules are in fact detected as local if they are installed
+        # as editable (as is common for contributors). Prevent the ast constant
+        # logic from picking up on them
+        for module in sys.modules.copy():
+            if module.startswith("hypofuzz"):
+                del sys.modules[module]
+
+        # local
+        a = 42
+        b = "test1"
+        c = True
+        d = 3.14
+        e = b"test2"
+        f = (1, 2)
+        g = frozenset([3, 4])
+        actual = local_constants()
+        assert actual == {
+            "hypofuzz",
+            42,
+            "test1",
+            True,
+            3.14,
+            b"test2",
+            1,
+            2,
+            3,
+            4
+        }, actual
+        """,
+        ),
+        encoding="utf-8",
+    )
+    subprocess.check_call([sys.executable, str(p)])
+
+
+def test_constants_from_bad_module():
+    # covering test for the except branch
+    module = ModuleType("nonexistent")
+    assert _module_ast(module) is None

tooling/src/hypothesistooling/projects/hypothesispython.py

@@ -221,7 +221,9 @@ def upload_distribution():
     # Construct plain-text + markdown version of this changelog entry,
     # with link to canonical source.
     build_docs(builder="text", only=["docs/changelog.rst"])
-    textfile = os.path.join(HYPOTHESIS_PYTHON, "docs", "_build", "text", "changes.txt")
+    textfile = os.path.join(
+        HYPOTHESIS_PYTHON, "docs", "_build", "text", "changelog.txt"
+    )
     with open(textfile, encoding="utf-8") as f:
         lines = f.readlines()
     entries = [i for i, l in enumerate(lines) if CHANGELOG_HEADER.match(l)]