raise flaky when replaying backend flakes

Author: tybug

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -38,7 +38,7 @@
 from hypothesis import HealthCheck, Phase, Verbosity, settings as Settings
 from hypothesis._settings import local_settings
 from hypothesis.database import ExampleDatabase
-from hypothesis.errors import InvalidArgument, StopTest
+from hypothesis.errors import Flaky, InvalidArgument, StopTest
 from hypothesis.internal.cache import LRUReusedCache
 from hypothesis.internal.compat import (
     NotRequired,
@@ -506,12 +506,20 @@ def test_function(self, data: ConjectureData) -> None:
             if self.settings.backend != "hypothesis":
                 # drive the ir tree through the test function to convert it
                 # to a buffer
+                initial_origin = data.interesting_origin
                 data = ConjectureData.for_ir_tree(data.examples.ir_tree_nodes)
                 self.__stoppable_test_function(data)
                 data.freeze()
-                # should we raise Flaky here instead?
+                # we'd like to use expected_failure machinery here from
+                # StateForActualGivenExecution for better diagnostic reports of eg
+                # flaky deadlines, but we're too low down in the engine for that.
+                # for now a worse generic flaky error will have to do.
                 if data.status != Status.INTERESTING:
-                    self.exit_with(ExitReason.flaky)
+                    raise Flaky(
+                        f"Inconsistent results from replaying a failing test case!\n"
+                        f"  last: {Status.INTERESTING.name} from {initial_origin}\n"
+                        f"  this: {data.status.name}"
+                    )
 
                 self._cache(data)
 

hypothesis-python/tests/conjecture/test_alt_backend.py

@@ -18,7 +18,7 @@
 
 from hypothesis import given, settings, strategies as st
 from hypothesis.database import InMemoryExampleDatabase
-from hypothesis.errors import InvalidArgument
+from hypothesis.errors import Flaky, InvalidArgument
 from hypothesis.internal.compat import int_to_bytes
 from hypothesis.internal.conjecture.data import (
     AVAILABLE_PROVIDERS,
@@ -354,3 +354,19 @@ def test_function(n):
             <= test_case_lifetime_init_count
             <= test_function_count + 10
         )
+
+
+def test_flaky_with_backend():
+    with temp_register_backend("trivial", TrivialProvider):
+
+        calls = 0
+
+        @given(st.integers())
+        @settings(backend="trivial", database=None)
+        def test_function(n):
+            nonlocal calls
+            calls += 1
+            assert n != calls % 2
+
+        with pytest.raises(Flaky):
+            test_function()