mark invalid on forced misalignment, add ir-specific comparison methods

Author: tybug

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -910,7 +910,7 @@ def draw_boolean(
         pass
 
 
-@attr.s(slots=True, repr=False)
+@attr.s(slots=True, repr=False, eq=False)
 class IRNode:
     ir_type: IRTypeName = attr.ib()
     value: IRType = attr.ib()
@@ -928,6 +928,17 @@ def copy(self, *, with_value: IRType) -> "IRNode":
             was_forced=self.was_forced,
         )
 
+    def __eq__(self, other):
+        if not isinstance(other, IRNode):
+            return NotImplemented
+
+        return (
+            self.ir_type == other.ir_type
+            and ir_value_equal(self.ir_type, self.value, other.value)
+            and ir_kwargs_equal(self.ir_type, self.kwargs, other.kwargs)
+            and self.was_forced == other.was_forced
+        )
+
     def __repr__(self):
         # repr to avoid "BytesWarning: str() on a bytes instance" for bytes nodes
         forced_marker = " [forced]" if self.was_forced else ""
@@ -967,6 +978,24 @@ def ir_value_permitted(value, ir_type, kwargs):
     raise NotImplementedError(f"unhandled type {type(value)} of ir value {value}")
 
 
+def ir_value_equal(ir_type, v1, v2):
+    if ir_type != "float":
+        return v1 == v2
+    return float_to_int(v1) == float_to_int(v2)
+
+
+def ir_kwargs_equal(ir_type, kwargs1, kwargs2):
+    if ir_type != "float":
+        return kwargs1 == kwargs2
+    return (
+        float_to_int(kwargs1["min_value"]) == float_to_int(kwargs2["min_value"])
+        and float_to_int(kwargs1["max_value"]) == float_to_int(kwargs2["max_value"])
+        and kwargs1["allow_nan"] == kwargs2["allow_nan"]
+        and kwargs1["smallest_nonzero_magnitude"]
+        == kwargs2["smallest_nonzero_magnitude"]
+    )
+
+
 @dataclass_transform()
 @attr.s(slots=True)
 class ConjectureResult:
@@ -1880,7 +1909,7 @@ def draw_integer(
         )
 
         if self.ir_tree_nodes is not None and observe:
-            node = self._pop_ir_tree_node("integer", kwargs)
+            node = self._pop_ir_tree_node("integer", kwargs, forced=forced)
             assert isinstance(node.value, int)
             forced = node.value
             fake_forced = not node.was_forced
@@ -1936,7 +1965,7 @@ def draw_float(
         )
 
         if self.ir_tree_nodes is not None and observe:
-            node = self._pop_ir_tree_node("float", kwargs)
+            node = self._pop_ir_tree_node("float", kwargs, forced=forced)
             assert isinstance(node.value, float)
             forced = node.value
             fake_forced = not node.was_forced
@@ -1977,7 +2006,7 @@ def draw_string(
             },
         )
         if self.ir_tree_nodes is not None and observe:
-            node = self._pop_ir_tree_node("string", kwargs)
+            node = self._pop_ir_tree_node("string", kwargs, forced=forced)
             assert isinstance(node.value, str)
             forced = node.value
             fake_forced = not node.was_forced
@@ -2012,7 +2041,7 @@ def draw_bytes(
         kwargs: BytesKWargs = self._pooled_kwargs("bytes", {"size": size})
 
         if self.ir_tree_nodes is not None and observe:
-            node = self._pop_ir_tree_node("bytes", kwargs)
+            node = self._pop_ir_tree_node("bytes", kwargs, forced=forced)
             assert isinstance(node.value, bytes)
             forced = node.value
             fake_forced = not node.was_forced
@@ -2053,7 +2082,7 @@ def draw_boolean(
         kwargs: BooleanKWargs = self._pooled_kwargs("boolean", {"p": p})
 
         if self.ir_tree_nodes is not None and observe:
-            node = self._pop_ir_tree_node("boolean", kwargs)
+            node = self._pop_ir_tree_node("boolean", kwargs, forced=forced)
             assert isinstance(node.value, bool)
             forced = node.value
             fake_forced = not node.was_forced
@@ -2093,7 +2122,9 @@ def _pooled_kwargs(self, ir_type, kwargs):
             POOLED_KWARGS_CACHE[key] = kwargs
             return kwargs
 
-    def _pop_ir_tree_node(self, ir_type: IRTypeName, kwargs: IRKWargsType) -> IRNode:
+    def _pop_ir_tree_node(
+        self, ir_type: IRTypeName, kwargs: IRKWargsType, *, forced: Optional[IRType]
+    ) -> IRNode:
         assert self.ir_tree_nodes is not None
 
         if self.ir_tree_nodes == []:
@@ -2120,6 +2151,32 @@ def _pop_ir_tree_node(self, ir_type: IRTypeName, kwargs: IRKWargsType) -> IRNode
         if not ir_value_permitted(node.value, node.ir_type, kwargs):
             self.mark_invalid()  # pragma: no cover  # FIXME @tybug
 
+        if forced is not None:
+            # if we expected a forced node but are instead returning a non-forced
+            # node, something has gone terribly wrong. If we allowed this combination,
+            # we risk violating core invariants that rely on forced draws being,
+            # well, forced to a particular value.
+            #
+            # In particular, this can manifest while shrinking. Consider the tree
+            #   [boolean True [forced] {"p": 0.5}]
+            #   [boolean False {"p": 0.5}]
+            #
+            # and the shrinker tries to reorder these to
+            #   [boolean False {"p": 0.5}]
+            #   [boolean True [forced] {"p": 0.5}].
+            #
+            # However, maybe we got lucky and the non-forced node is returning
+            # the same value that was expected from the forced draw. We lucked
+            # into an aligned tree in this case and can let it slide.
+            if not node.was_forced and not ir_value_equal(ir_type, forced, node.value):
+                self.mark_invalid()
+
+            # similarly, if we expected a forced node with a certain value, and
+            # are returning a forced node with a different value, this is an
+            # equally bad misalignment.
+            if node.was_forced and not ir_value_equal(ir_type, forced, node.value):
+                self.mark_invalid()
+
         return node
 
     def as_result(self) -> Union[ConjectureResult, _Overrun]:

hypothesis-python/tests/conjecture/test_ir.py

@@ -9,15 +9,17 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import math
+from copy import deepcopy
 
 import pytest
 
-from hypothesis import assume, example, given, strategies as st
+from hypothesis import HealthCheck, assume, example, given, settings, strategies as st
 from hypothesis.errors import StopTest
 from hypothesis.internal.conjecture.data import (
     ConjectureData,
     IRNode,
     Status,
+    ir_value_equal,
     ir_value_permitted,
 )
 from hypothesis.internal.conjecture.datatree import (
@@ -328,13 +330,12 @@ def test_ir_nodes(random):
 
 
 @st.composite
-def ir_nodes(draw):
+def ir_nodes(draw, *, was_forced=None):
     (ir_type, kwargs) = draw(ir_types_and_kwargs())
     value = draw_value(ir_type, kwargs)
+    was_forced = draw(st.booleans()) if was_forced is None else was_forced
 
-    return IRNode(
-        ir_type=ir_type, value=value, kwargs=kwargs, was_forced=draw(st.booleans())
-    )
+    return IRNode(ir_type=ir_type, value=value, kwargs=kwargs, was_forced=was_forced)
 
 
 @given(ir_nodes())
@@ -343,11 +344,17 @@ def test_copy_ir_node(node):
 
     assume(not node.was_forced)
     new_value = draw_value(node.ir_type, node.kwargs)
-    # if we drew the same value as before, the node should still be equal (unless nan)
-    assume(
-        node.ir_type != "float" or not (math.isnan(new_value) or math.isnan(node.value))
+    # if we drew the same value as before, the node should still be equal
+    assert (node.copy(with_value=new_value) == node) is (
+        ir_value_equal(node.ir_type, new_value, node.value)
     )
-    assert (node.copy(with_value=new_value) == node) is (new_value == node.value)
+
+
+@given(ir_nodes())
+def test_ir_node_equality(node):
+    assert node == node
+    # for coverage on our NotImplemented return, more than anything.
+    assert node != 42
 
 
 def test_data_with_empty_ir_tree_is_overrun():
@@ -378,6 +385,60 @@ def test_data_with_misaligned_ir_tree_is_invalid(data):
     assert data.status is Status.INVALID
 
 
+@given(st.data())
+def test_data_with_changed_was_forced_is_invalid(data):
+    # we had a normal node and then tried to draw a different forced value from it.
+    # ir tree: v1 [was_forced=False]
+    # drawing:    [forced=v2]
+    node = data.draw(ir_nodes(was_forced=False))
+    data = ConjectureData.for_ir_tree([node])
+
+    draw_func = getattr(data, f"draw_{node.ir_type}")
+    kwargs = deepcopy(node.kwargs)
+    kwargs["forced"] = draw_value(node.ir_type, node.kwargs)
+    assume(not ir_value_equal(node.ir_type, kwargs["forced"], node.value))
+
+    with pytest.raises(StopTest):
+        draw_func(**kwargs)
+
+    assert data.status is Status.INVALID
+
+
+@given(st.data())
+@settings(suppress_health_check=[HealthCheck.too_slow])
+def test_data_with_changed_forced_value_is_invalid(data):
+    # we had a forced node and then tried to draw a different forced value from it.
+    # ir tree: v1 [was_forced=True]
+    # drawing:    [forced=v2]
+    node = data.draw(ir_nodes(was_forced=True))
+    data = ConjectureData.for_ir_tree([node])
+
+    draw_func = getattr(data, f"draw_{node.ir_type}")
+    kwargs = deepcopy(node.kwargs)
+    kwargs["forced"] = draw_value(node.ir_type, node.kwargs)
+    assume(not ir_value_equal(node.ir_type, kwargs["forced"], node.value))
+
+    with pytest.raises(StopTest):
+        draw_func(**kwargs)
+
+    assert data.status is Status.INVALID
+
+
+@given(st.data())
+def test_data_with_same_forced_value_is_valid(data):
+    # we had a forced node and then drew the same forced value. This is totally
+    # fine!
+    # ir tree: v1 [was_forced=True]
+    # drawing:    [forced=v1]
+    node = data.draw(ir_nodes(was_forced=True))
+    data = ConjectureData.for_ir_tree([node])
+    draw_func = getattr(data, f"draw_{node.ir_type}")
+
+    kwargs = deepcopy(node.kwargs)
+    kwargs["forced"] = node.value
+    draw_func(**kwargs)
+
+
 @given(ir_types_and_kwargs())
 def test_all_children_are_permitted_values(ir_type_and_kwargs):
     (ir_type, kwargs) = ir_type_and_kwargs