Clean up some backend-dependant tests

Author: Zac-HD

hypothesis-python/tests/cover/test_core.py

@@ -80,18 +80,20 @@ def inner(x):
 
 
 def test_pytest_skip_skips_shrinking():
-    values = []
+    seen_large = False
 
     @settings(derandomize=True, max_examples=100)
     @given(s.integers())
     def inner(x):
-        values.append(x)
+        nonlocal seen_large
         if x > 100:
+            if seen_large:
+                raise Exception("Should never replay a skipped test!")
+            seen_large = True
             pytest.skip(f"{x=} is too big!")
 
     with pytest.raises(Skipped):
         inner()
-    assert len([x for x in values if x > 100]) == 1
 
 
 def test_can_find_with_db_eq_none():

hypothesis-python/tests/cover/test_deadline.py

@@ -53,7 +53,7 @@ def test_slow_with_none_deadline(i):
 def test_raises_flaky_if_a_test_becomes_fast_on_rerun():
     once = [True]
 
-    @settings(deadline=500)
+    @settings(deadline=500, backend="hypothesis")
     @given(st.integers())
     def test_flaky_slow(i):
         if once[0]:
@@ -82,11 +82,10 @@ def test_keeps_you_well_above_the_deadline():
     seen = set()
     failed_once = [False]
 
-    @settings(deadline=100)
+    @settings(deadline=100, backend="hypothesis")
     @given(st.integers(0, 2000))
     def slow(i):
-        # Make sure our initial failure isn't something that immediately goes
-        # flaky.
+        # Make sure our initial failure isn't something that immediately goes flaky.
         if not failed_once[0]:
             if i * 0.9 <= 100:
                 return
@@ -107,7 +106,7 @@ def slow(i):
 def test_gives_a_deadline_specific_flaky_error_message():
     once = [True]
 
-    @settings(deadline=100)
+    @settings(deadline=100, backend="hypothesis")
     @given(st.integers())
     def slow_once(i):
         if once[0]:

hypothesis-python/tests/cover/test_health_checks.py

@@ -8,13 +8,13 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import re
 import time
 
 import pytest
-from pytest import raises
 
 from hypothesis import HealthCheck, given, settings, strategies as st
-from hypothesis.control import assume
+from hypothesis.control import assume, current_build_context
 from hypothesis.errors import FailedHealthCheck, InvalidArgument
 from hypothesis.internal.compat import int_from_bytes
 from hypothesis.stateful import (
@@ -28,26 +28,28 @@
 
 from tests.common.utils import no_shrink
 
-HEALTH_CHECK_SETTINGS = settings(max_examples=11, database=None)
+HEALTH_CHECK_SETTINGS = settings(
+    max_examples=11, database=None, suppress_health_check=()
+)
 
 
 def test_slow_generation_fails_a_health_check():
-    @HEALTH_CHECK_SETTINGS
+    @settings(HEALTH_CHECK_SETTINGS, deadline=200)
     @given(st.integers().map(lambda x: time.sleep(0.2)))
     def test(x):
         pass
 
-    with raises(FailedHealthCheck):
+    with pytest.raises(FailedHealthCheck):
         test()
 
 
 def test_slow_generation_inline_fails_a_health_check():
-    @HEALTH_CHECK_SETTINGS
+    @settings(HEALTH_CHECK_SETTINGS, deadline=200)
     @given(st.data())
     def test(data):
         data.draw(st.integers().map(lambda x: time.sleep(0.2)))
 
-    with raises(FailedHealthCheck):
+    with pytest.raises(FailedHealthCheck):
         test()
 
 
@@ -75,7 +77,7 @@ def unhealthy_filter(x):
     def test1(x):
         raise ValueError
 
-    with raises(FailedHealthCheck):
+    with pytest.raises(FailedHealthCheck):
         test1()
 
     forbidden = set()
@@ -85,7 +87,7 @@ def test1(x):
     def test2(x):
         raise ValueError
 
-    with raises(ValueError):
+    with pytest.raises(ValueError):
         test2()
 
 
@@ -95,9 +97,8 @@ def test_filtering_everything_fails_a_health_check():
     def test(x):
         pass
 
-    with raises(FailedHealthCheck) as e:
+    with pytest.raises(FailedHealthCheck, match="filter"):
         test()
-    assert "filter" in e.value.args[0]
 
 
 class fails_regularly(SearchStrategy):
@@ -109,21 +110,21 @@ def do_draw(self, data):
 
 def test_filtering_most_things_fails_a_health_check():
     @given(fails_regularly())
-    @settings(database=None, phases=no_shrink)
+    @settings(database=None, phases=no_shrink, suppress_health_check=())
     def test(x):
-        pass
+        if current_build_context().data.provider.avoid_realization:
+            pytest.skip("symbolic backends can filter efficiently!")
 
-    with raises(FailedHealthCheck) as e:
+    with pytest.raises(FailedHealthCheck, match="filter"):
         test()
-    assert "filter" in e.value.args[0]
 
 
 def test_returning_non_none_is_forbidden():
     @given(st.integers())
     def a(x):
         return 1
 
-    with raises(FailedHealthCheck):
+    with pytest.raises(FailedHealthCheck):
         a()
 
 
@@ -153,19 +154,18 @@ def test(self, _):
 
 def test_differing_executors_fails_health_check():
     sample_test_runner().test()
-    with pytest.raises(FailedHealthCheck) as exc:
+    msg = re.escape(str(HealthCheck.differing_executors))
+    with pytest.raises(FailedHealthCheck, match=msg):
         sample_test_runner().test()
 
-    assert str(HealthCheck.differing_executors) in str(exc.value)
-
 
 def test_it_is_an_error_to_suppress_non_iterables():
-    with raises(InvalidArgument):
+    with pytest.raises(InvalidArgument):
         settings(suppress_health_check=1)
 
 
 def test_it_is_an_error_to_suppress_non_healthchecks():
-    with raises(InvalidArgument):
+    with pytest.raises(InvalidArgument):
         settings(suppress_health_check=[1])
 
 

hypothesis-python/tests/cover/test_targeting.py

@@ -13,6 +13,7 @@
 import pytest
 
 from hypothesis import example, given, strategies as st, target
+from hypothesis.control import current_build_context
 from hypothesis.errors import InvalidArgument
 
 
@@ -92,6 +93,8 @@ def test_cannot_target_outside_test():
 
 @given(st.none())
 def test_cannot_target_same_label_twice(_):
+    if current_build_context().data.provider.avoid_realization:
+        pytest.skip("target() is a noop to avoid realizing arguments")
     target(0.0, label="label")
     with pytest.raises(InvalidArgument):
         target(1.0, label="label")

hypothesis-python/tests/nocover/test_randomization.py

@@ -8,7 +8,7 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
-from pytest import raises
+import pytest
 
 from hypothesis import Verbosity, core, find, given, settings, strategies as st
 
@@ -35,7 +35,7 @@ def test_blah(x, rnd):
         def test_nest(y):
             assert y < x
 
-        with raises(AssertionError):
+        with pytest.raises(AssertionError):
             test_nest()
 
     test_blah()