Merge pull request #4163 from tybug/optimiser-ir

tybug · web-flow · commit d863cb1c7021 · 2024-11-09T21:50:41.000-05:00
Migrate `Optimiser` to the typed choice sequence
diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst
@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch migrates the optimisation algorithm for :ref:`targeted property-based testing <targeted-search>` to our IR layer (:issue:`3921`). This should result in moderately different (and hopefully improved) exploration behavior in tests which use :func:`hypothesis.target`.
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/optimiser.py b/hypothesis-python/src/hypothesis/internal/conjecture/optimiser.py
@@ -11,8 +11,16 @@
 from typing import Union
 
 from hypothesis.internal.compat import int_from_bytes, int_to_bytes
-from hypothesis.internal.conjecture.data import ConjectureResult, Status, _Overrun
-from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner
+from hypothesis.internal.conjecture.data import (
+    ConjectureResult,
+    IRType,
+    Status,
+    _Overrun,
+    bits_to_bytes,
+    ir_size_nodes,
+    ir_value_permitted,
+)
+from hypothesis.internal.conjecture.engine import BUFFER_SIZE_IR, ConjectureRunner
 from hypothesis.internal.conjecture.junkdrawer import find_integer
 from hypothesis.internal.conjecture.pareto import NO_SCORE
 
@@ -75,9 +83,11 @@ def consider_new_data(self, data: Union[ConjectureResult, _Overrun]) -> bool:
             return True
         assert score == self.current_score
         # We allow transitions that leave the score unchanged as long as they
-        # don't increase the buffer size. This gives us a certain amount of
+        # don't increase the number of nodes. This gives us a certain amount of
         # freedom for lateral moves that will take us out of local maxima.
-        if len(data.buffer) <= len(self.current_data.buffer):
+        if len(data.examples.ir_tree_nodes) <= len(
+            self.current_data.examples.ir_tree_nodes
+        ):
             self.current_data = data
             return True
         return False
@@ -88,92 +98,123 @@ def hill_climb(self) -> None:
         a data object and returns an index to an example where we should focus
         our efforts."""
 
-        blocks_examined = set()
+        nodes_examined = set()
 
         prev = None
-        i = len(self.current_data.blocks) - 1
+        i = len(self.current_data.examples.ir_tree_nodes) - 1
         while i >= 0 and self.improvements <= self.max_improvements:
             if prev is not self.current_data:
-                i = len(self.current_data.blocks) - 1
+                i = len(self.current_data.examples.ir_tree_nodes) - 1
                 prev = self.current_data
 
-            if i in blocks_examined:
+            if i in nodes_examined:
                 i -= 1
                 continue
 
-            blocks_examined.add(i)
-            data = self.current_data
-            block = data.blocks[i]
-            prefix = data.buffer[: block.start]
+            nodes_examined.add(i)
+            node = self.current_data.examples.ir_tree_nodes[i]
+            assert node.index is not None
+            # we can only (sensibly & easily) define hill climbing for
+            # numeric-style nodes. It's not clear hill-climbing a string is
+            # useful, for instance.
+            if node.ir_type not in {"integer", "float", "bytes", "boolean"}:
+                continue
 
-            existing = data.buffer[block.start : block.end]
-            existing_as_int = int_from_bytes(existing)
-            max_int_value = (256 ** len(existing)) - 1
+            def attempt_replace(k: int) -> bool:
+                """
+                Try replacing the current node in the current best test case
+                with a value which is "k times larger", where the exact notion
+                of "larger" depends on the ir_type.
+
+                Note that we use the *current* best and not the one we started with.
+                This helps ensure that if we luck into a good draw when making
+                random choices we get to keep the good bits.
+                """
+                # we don't want to infinitely drive up an unbounded score.
+                if abs(k) > 2**20:
+                    return False
 
-            if existing_as_int == max_int_value:
-                continue
+                node = self.current_data.examples.ir_tree_nodes[i]
+                assert node.index is not None
+                if node.was_forced:
+                    return False  # pragma: no cover
+
+                new_value: IRType
+                if node.ir_type in {"integer", "float"}:
+                    assert isinstance(node.value, (int, float))
+                    new_value = node.value + k
+                elif node.ir_type == "boolean":
+                    assert isinstance(node.value, bool)
+                    if abs(k) > 1:
+                        return False
+                    if k == -1:
+                        new_value = False
+                    if k == 1:
+                        new_value = True
+                    if k == 0:  # pragma: no cover
+                        new_value = node.value
+                else:
+                    assert node.ir_type == "bytes"
+                    assert isinstance(node.value, bytes)
+                    v = int_from_bytes(node.value)
+                    # can't go below zero for bytes
+                    if v + k < 0:
+                        return False
+                    v += k
+                    # allow adding k to increase the number of bytes. we don't want
+                    # to decrease so that b"01" doesn't turn into b"1".
+                    size = max(len(node.value), bits_to_bytes(v.bit_length()))
+                    new_value = int_to_bytes(v, size)
 
-            def attempt_replace(v: int) -> bool:
-                """Try replacing the current block in the current best test case
-                 with an integer of value i. Note that we use the *current*
-                best and not the one we started with. This helps ensure that
-                if we luck into a good draw when making random choices we get
-                to keep the good bits."""
-                if v < 0 or v > max_int_value:
+                if not ir_value_permitted(new_value, node.ir_type, node.kwargs):
                     return False
-                v_as_bytes = int_to_bytes(v, len(existing))
 
-                # We make a couple attempts at replacement. This only matters
-                # if we end up growing the buffer - otherwise we exit the loop
-                # early - but in the event that there *is* some randomized
-                # component we want to give it a couple of tries to succeed.
                 for _ in range(3):
-                    attempt = self.engine.cached_test_function(
-                        prefix
-                        + v_as_bytes
-                        + self.current_data.buffer[block.end :]
-                        + bytes(BUFFER_SIZE),
+                    nodes = self.current_data.examples.ir_tree_nodes
+                    attempt_nodes = (
+                        nodes[: node.index]
+                        + (node.copy(with_value=new_value),)
+                        + nodes[node.index + 1 :]
+                    )
+                    attempt = self.engine.cached_test_function_ir(
+                        attempt_nodes,
+                        extend=BUFFER_SIZE_IR - ir_size_nodes(attempt_nodes),
                     )
 
                     if self.consider_new_data(attempt):
                         return True
 
-                    if attempt.status == Status.OVERRUN:
+                    if attempt.status is Status.OVERRUN:
                         return False
 
                     assert isinstance(attempt, ConjectureResult)
-                    if len(attempt.buffer) == len(self.current_data.buffer):
+                    if len(attempt.examples.ir_tree_nodes) == len(
+                        self.current_data.examples.ir_tree_nodes
+                    ):
                         return False
 
-                    for i, ex in enumerate(self.current_data.examples):
-                        if ex.start >= block.end:
+                    for j, ex in enumerate(self.current_data.examples):
+                        if ex.ir_start >= node.index + 1:
                             break  # pragma: no cover
-                        if ex.end <= block.start:
+                        if ex.ir_end <= node.index:
                             continue
-                        ex_attempt = attempt.examples[i]
-                        if ex.length == ex_attempt.length:
+                        ex_attempt = attempt.examples[j]
+                        if ex.ir_length == ex_attempt.ir_length:
                             continue  # pragma: no cover
-                        replacement = attempt.buffer[ex_attempt.start : ex_attempt.end]
+                        replacement = attempt.examples.ir_tree_nodes[
+                            ex_attempt.ir_start : ex_attempt.ir_end
+                        ]
                         if self.consider_new_data(
-                            self.engine.cached_test_function(
-                                prefix
+                            self.engine.cached_test_function_ir(
+                                nodes[: node.index]
                                 + replacement
-                                + self.current_data.buffer[ex.end :]
+                                + self.current_data.examples.ir_tree_nodes[ex.ir_end :]
                             )
                         ):
                             return True
                 return False
 
-            # We unconditionally scan both upwards and downwards. The reason
-            # for this is that we allow "lateral" moves that don't increase the
-            # score but instead leave it constant. All else being equal we'd
-            # like to leave the test case closer to shrunk, so afterwards we
-            # try lowering the value towards zero even if we've just raised it.
-
-            if not attempt_replace(max_int_value):
-                find_integer(lambda k: attempt_replace(k + existing_as_int))
-
-            existing = self.current_data.buffer[block.start : block.end]
-            existing_as_int = int_from_bytes(existing)
-            if not attempt_replace(0):
-                find_integer(lambda k: attempt_replace(existing_as_int - k))
+            # we don't know whether a target score increases or decreases with
+            # respect to the value of some node, so try both directions.
+            find_integer(lambda k: attempt_replace(k))
+            find_integer(lambda k: attempt_replace(-k))
diff --git a/hypothesis-python/tests/conjecture/test_optimiser.py b/hypothesis-python/tests/conjecture/test_optimiser.py
@@ -8,15 +8,18 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import math
+
 import pytest
 
-from hypothesis import settings
+from hypothesis import assume, example, given, settings
 from hypothesis.internal.compat import int_to_bytes
-from hypothesis.internal.conjecture.data import Status
+from hypothesis.internal.conjecture.data import IRNode, Status
+from hypothesis.internal.conjecture.datatree import compute_max_children
 from hypothesis.internal.conjecture.engine import ConjectureRunner, RunIsComplete
 from hypothesis.internal.entropy import deterministic_PRNG
 
-from tests.conjecture.common import TEST_SETTINGS, buffer_size_limit
+from tests.conjecture.common import TEST_SETTINGS, buffer_size_limit, ir_nodes
 
 
 def test_optimises_to_maximum():
@@ -219,3 +222,38 @@ def test(data):
                 pass
 
             assert runner.best_observed_targets["m"] == 100
+
+
+@given(ir_nodes())
+@example(
+    IRNode(
+        ir_type="bytes",
+        value=b"\xb1",
+        kwargs={"min_size": 1, "max_size": 1},
+        was_forced=False,
+    )
+)
+def test_optimising_all_nodes(node):
+    assume(compute_max_children(node.ir_type, node.kwargs) > 100)
+    size_function = {
+        "integer": lambda n: n,
+        "float": lambda f: f if math.isfinite(f) else 0,
+        "string": lambda s: len(s),
+        "bytes": lambda b: len(b),
+        "boolean": lambda b: int(b),
+    }
+    with deterministic_PRNG():
+
+        def test(data):
+            v = getattr(data, f"draw_{node.ir_type}")(**node.kwargs)
+            data.target_observations["v"] = size_function[node.ir_type](v)
+
+        runner = ConjectureRunner(
+            test, settings=settings(TEST_SETTINGS, max_examples=50)
+        )
+        runner.cached_test_function_ir([node])
+
+        try:
+            runner.optimise_targets()
+        except RunIsComplete:
+            pass

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+RELEASE_TYPE: patch`
	`2`	`+`
	`3`	+This patch migrates the optimisation algorithm for :ref:`targeted property-based testing <targeted-search>` to our IR layer (:issue:`3921`). This should result in moderately different (and hopefully improved) exploration behavior in tests which use :func:`hypothesis.target`.