Merge pull request #4047 from Zac-HD/various-ci-improvements

Test-only improvements

Author: Zac-HD

hypothesis-python/setup.py

@@ -60,7 +60,7 @@ def local_file(name):
     "pytest": ["pytest>=4.6"],
     "dpcontracts": ["dpcontracts>=0.4"],
     "redis": ["redis>=3.0.0"],
-    "crosshair": ["hypothesis-crosshair>=0.0.7", "crosshair-tool>=0.0.59"],
+    "crosshair": ["hypothesis-crosshair>=0.0.7", "crosshair-tool>=0.0.60"],
     # zoneinfo is an odd one: every dependency is conditional, because they're
     # only necessary on old versions of Python or Windows systems or emscripten.
     "zoneinfo": [

hypothesis-python/tests/array_api/test_arrays.py

@@ -133,7 +133,8 @@ def test_generate_arrays_from_zero_dimensions(xp, xps):
 def test_generate_arrays_from_zero_sided_shapes(xp, xps, data):
     """Generate arrays from shapes with at least one 0-sized dimension."""
     shape = data.draw(xps.array_shapes(min_side=0).filter(lambda s: 0 in s))
-    assert_all_examples(xps.arrays(xp.int8, shape), lambda x: x.shape == shape)
+    arr = data.draw(xps.arrays(xp.int8, shape))
+    assert arr.shape == shape
 
 
 def test_generate_arrays_from_unsigned_ints(xp, xps):

hypothesis-python/tests/common/debug.py

@@ -8,7 +8,10 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+from unittest import SkipTest
+
 from hypothesis import HealthCheck, Phase, Verbosity, given, settings as Settings
+from hypothesis.control import _current_build_context
 from hypothesis.errors import Found, NoSuchExample, Unsatisfiable
 from hypothesis.internal.reflection import get_pretty_function_description
 
@@ -26,6 +29,11 @@ def minimal(definition, condition=lambda x: True, settings=None, timeout_after=1
     runtime = None
     result = None
 
+    if (
+        context := _current_build_context.value
+    ) and context.data.provider.avoid_realization:
+        raise SkipTest("`minimal()` helper not supported under symbolic execution")
+
     def wrapped_condition(x):
         nonlocal runtime
         if timeout_after is not None:
@@ -71,6 +79,13 @@ def inner(x):
 
 
 def find_any(definition, condition=lambda _: True, settings=None):
+    # If nested within an existing @given
+    if context := _current_build_context.value:
+        while True:
+            if condition(s := context.data.draw(definition)):
+                return s
+
+    # If top-level
     settings = settings or Settings.default
     return minimal(
         definition,
@@ -83,8 +98,7 @@ def find_any(definition, condition=lambda _: True, settings=None):
 
 def assert_no_examples(strategy, condition=lambda _: True):
     try:
-        result = find_any(strategy, condition)
-        raise AssertionError(f"Expected no results but found {result!r}")
+        assert_all_examples(strategy, lambda val: not condition(val))
     except (Unsatisfiable, NoSuchExample):
         pass
 
@@ -95,14 +109,21 @@ def assert_all_examples(strategy, predicate, settings=None):
     :param strategy: Hypothesis strategy to check
     :param predicate: (callable) Predicate that takes example and returns bool
     """
+    if context := _current_build_context.value:
+        for _ in range(20):
+            s = context.data.draw(strategy)
+            msg = f"Found {s!r} using strategy {strategy} which does not match"
+            assert predicate(s), msg
+
+    else:
 
-    @given(strategy)
-    @Settings(parent=settings, database=None)
-    def assert_examples(s):
-        msg = f"Found {s!r} using strategy {strategy} which does not match"
-        assert predicate(s), msg
+        @given(strategy)
+        @Settings(parent=settings, database=None)
+        def assert_examples(s):
+            msg = f"Found {s!r} using strategy {strategy} which does not match"
+            assert predicate(s), msg
 
-    assert_examples()
+        assert_examples()
 
 
 def assert_simple_property(strategy, predicate, settings=None):

hypothesis-python/tests/common/setup.py

@@ -11,7 +11,7 @@
 import os
 from warnings import filterwarnings
 
-from hypothesis import Phase, Verbosity, settings
+from hypothesis import HealthCheck, Phase, Verbosity, settings
 from hypothesis._settings import not_set
 from hypothesis.internal.conjecture.data import AVAILABLE_PROVIDERS
 from hypothesis.internal.coverage import IN_COVERAGE_TESTS
@@ -61,7 +61,17 @@ def run():
 
     settings.register_profile("debug", settings(verbosity=Verbosity.debug))
 
-    for backend in set(AVAILABLE_PROVIDERS) - {"hypothesis"}:
-        settings.register_profile(backend, backend=backend)  # e.g. "crosshair"
+    if "crosshair" in AVAILABLE_PROVIDERS:
+        settings.register_profile(
+            "crosshair",
+            backend="crosshair",
+            max_examples=20,
+            deadline=None,
+            suppress_health_check=list(HealthCheck),
+            report_multiple_bugs=False,
+        )
+
+    for backend in set(AVAILABLE_PROVIDERS) - {"hypothesis", "crosshair"}:
+        settings.register_profile(backend, backend=backend)
 
     settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default"))

hypothesis-python/tests/cover/test_datetimes.py

@@ -12,7 +12,7 @@
 
 import pytest
 
-from hypothesis import given, settings
+from hypothesis import HealthCheck, given, settings
 from hypothesis.strategies import dates, datetimes, timedeltas, times
 
 from tests.common.debug import assert_simple_property, find_any, minimal
@@ -49,6 +49,7 @@ def test_max_value_is_respected():
     assert minimal(timedeltas(max_value=dt.timedelta(days=-10))).days == -10
 
 
+@settings(suppress_health_check=list(HealthCheck))
 @given(timedeltas())
 def test_single_timedelta(val):
     assert_simple_property(timedeltas(val, val), lambda v: v is val)