Merge pull request #3991 from tybug/novel-prefix-overrun

Handle overruns in `generate_novel_prefix`

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch fixes a rare internal error when generating very large elements from strategies (:issue:`3874`).

hypothesis-python/src/hypothesis/internal/conjecture/datatree.py

@@ -755,7 +755,14 @@ def append_buf(buf):
                     attempts = 0
                     while True:
                         if attempts <= 10:
-                            (v, buf) = self._draw(ir_type, kwargs, random=random)
+                            try:
+                                (v, buf) = self._draw(ir_type, kwargs, random=random)
+                            except StopTest:  # pragma: no cover
+                                # it is possible that drawing from a fresh data can
+                                # overrun BUFFER_SIZE, due to eg unlucky rejection sampling
+                                # of integer probes. Retry these cases.
+                                attempts += 1
+                                continue
                         else:
                             (v, buf) = self._draw_from_cache(
                                 ir_type, kwargs, key=id(current_node), random=random
@@ -781,9 +788,13 @@ def append_buf(buf):
                 attempts = 0
                 while True:
                     if attempts <= 10:
-                        (v, buf) = self._draw(
-                            branch.ir_type, branch.kwargs, random=random
-                        )
+                        try:
+                            (v, buf) = self._draw(
+                                branch.ir_type, branch.kwargs, random=random
+                            )
+                        except StopTest:  # pragma: no cover
+                            attempts += 1
+                            continue
                     else:
                         (v, buf) = self._draw_from_cache(
                             branch.ir_type, branch.kwargs, key=id(branch), random=random

hypothesis-python/tests/conjecture/test_engine.py

@@ -1626,3 +1626,38 @@ def test(cd):
 
     runner.cached_test_function_ir([node] if forced_first else [forced_node])
     assert runner.call_count == 1
+
+
+def test_simulate_to_evicted_data(monkeypatch):
+    # test that we do not rely on the false invariant that correctly simulating
+    # a data to a result means we have that result in the cache, due to e.g.
+    # cache evictions (but also potentially other trickery).
+    monkeypatch.setattr(engine_module, "CACHE_SIZE", 1)
+
+    node_0 = IRNode(
+        ir_type="integer",
+        value=0,
+        kwargs={
+            "min_value": None,
+            "max_value": None,
+            "weights": None,
+            "shrink_towards": 0,
+        },
+        was_forced=False,
+    )
+    node_1 = node_0.copy(with_value=1)
+
+    def test(data):
+        data.draw_integer()
+
+    runner = ConjectureRunner(test)
+    runner.cached_test_function_ir([node_0])
+    # cache size is 1 so this evicts node_0
+    runner.cached_test_function_ir([node_1])
+    assert runner.call_count == 2
+
+    # we dont throw PreviouslyUnseenBehavior when simulating, but the result
+    # was evicted to the cache so we will still call through to the test function.
+    runner.tree.simulate_test_function(ConjectureData.for_ir_tree([node_0]))
+    runner.cached_test_function_ir([node_0])
+    assert runner.call_count == 3

hypothesis-python/tests/nocover/test_regressions.py

@@ -46,7 +46,6 @@ def test_performance_issue_2027(x, y):
     pass
 
 
-# if this test ever fails with a flaky error, #3926 has regressed.
 @given(
     st.lists(
         st.floats(allow_infinity=False),
@@ -55,3 +54,10 @@ def test_performance_issue_2027(x, y):
 )
 def test_unique_floats_with_nan_is_not_flaky_3926(ls):
     pass
+
+
+# this will take a while to find the regression, but will eventually trigger it.
+# min_value=0 is critical to trigger the probing behavior which exhausts our buffer.
+@given(st.integers(min_value=0, max_value=1 << 25_000))
+def test_overrun_during_datatree_simulation_3874(n):
+    pass