Avoid silently skipped tests

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,7 @@
+RELEASE_TYPE: minor
+
+This release raises :class:`~unittest.SkipTest` for which never executed any
+examples, for example because the :obj:`~hypothesis.settings.phases` setting
+excluded the :obj:`~hypothesis.Phase.explicit`, :obj:`~hypothesis.Phase.reuse`,
+and :obj:`~hypothesis.Phase.generate` phases.  This helps to avoid cases where
+broken tests appear to pass, because they didn't actually execute (:issue:`3328`).

hypothesis-python/src/hypothesis/core.py

@@ -18,6 +18,7 @@
 import sys
 import time
 import types
+import unittest
 import warnings
 import zlib
 from collections import defaultdict
@@ -556,6 +557,7 @@ def __init__(
         self.__was_flaky = False
         self.random = random
         self.__test_runtime = None
+        self.ever_executed = False
 
         self.is_find = getattr(wrapped_test, "_hypothesis_internal_is_find", False)
         self.wrapped_test = wrapped_test
@@ -586,6 +588,7 @@ def execute_once(
         swallowed the corresponding control exception.
         """
 
+        self.ever_executed = True
         data.is_find = self.is_find
 
         text_repr = None
@@ -1189,9 +1192,17 @@ def wrapped_test(*arguments, **kwargs):
             # The next step is to use the Conjecture engine to run the test on
             # many different inputs.
 
+            ran_explicit_examples = Phase.explicit in state.settings.phases and getattr(
+                wrapped_test, "hypothesis_explicit_examples", ()
+            )
+            SKIP_BECAUSE_NO_EXAMPLES = unittest.SkipTest(
+                "Hypothesis has been told to run no examples for this test."
+            )
             if not (
                 Phase.reuse in settings.phases or Phase.generate in settings.phases
             ):
+                if not ran_explicit_examples:
+                    raise SKIP_BECAUSE_NO_EXAMPLES
                 return
 
             try:
@@ -1236,6 +1247,9 @@ def wrapped_test(*arguments, **kwargs):
                     )
                     raise the_error_hypothesis_found
 
+            if not (ran_explicit_examples or state.ever_executed):
+                raise SKIP_BECAUSE_NO_EXAMPLES
+
         def _get_fuzz_target() -> Callable[
             [Union[bytes, bytearray, memoryview, BinaryIO]], Optional[bytes]
         ]:

hypothesis-python/tests/cover/test_core.py

@@ -8,10 +8,13 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import unittest
+
 import pytest
 from _pytest.outcomes import Failed, Skipped
 
-from hypothesis import find, given, reject, settings, strategies as s
+from hypothesis import Phase, example, find, given, reject, settings, strategies as s
+from hypothesis.database import InMemoryExampleDatabase
 from hypothesis.errors import InvalidArgument, NoSuchExample, Unsatisfiable
 
 
@@ -111,3 +114,32 @@ def test_method_with_bad_strategy(self, x):
     instance = TestStrategyValidation()
     with pytest.raises(InvalidArgument):
         instance.test_method_with_bad_strategy()
+
+
+@example(1)
+@given(s.integers())
+@settings(phases=[Phase.target, Phase.shrink, Phase.explain])
+def no_phases(_):
+    raise Exception
+
+
+@given(s.integers())
+@settings(phases=[Phase.explicit])
+def no_explicit(_):
+    raise Exception
+
+
+@given(s.integers())
+@settings(phases=[Phase.reuse], database=InMemoryExampleDatabase())
+def empty_db(_):
+    raise Exception
+
+
+@pytest.mark.parametrize(
+    "test_fn",
+    [no_phases, no_explicit, empty_db],
+    ids=lambda t: t.__name__,
+)
+def test_non_executed_tests_raise_skipped(test_fn):
+    with pytest.raises(unittest.SkipTest):
+        test_fn()

hypothesis-python/tests/cover/test_fuzz_one_input.py

@@ -9,6 +9,7 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import io
+import unittest
 from operator import attrgetter
 
 import pytest
@@ -45,7 +46,8 @@ def test(s):
 
     # Before running fuzz_one_input, there's nothing in `db`, and so the test passes
     # (because example generation is disabled by the custom settings)
-    test()
+    with pytest.raises(unittest.SkipTest):  # because this generates no examples
+        test()
     assert len(seen) == 0
 
     # If we run a lot of random bytestrings through fuzz_one_input, we'll eventually