Merge pull request #3455 from Zac-HD/example-consistency

Consistent skipping from explicit examples

Author: Zac-HD

build.sh

@@ -23,8 +23,8 @@ if [ -n "${GITHUB_ACTIONS-}" ] || [ -n "${CODESPACES-}" ] ; then
 else
     # Otherwise, we install it from scratch
     # NOTE: tooling keeps this version in sync with ci_version in tooling
-    "$SCRIPTS/ensure-python.sh" 3.8.13
-    PYTHON=$(pythonloc 3.8.13)/bin/python
+    "$SCRIPTS/ensure-python.sh" 3.8.14
+    PYTHON=$(pythonloc 3.8.14)/bin/python
 fi
 
 TOOL_REQUIREMENTS="$ROOT/requirements/tools.txt"

hypothesis-python/.coveragerc

@@ -19,6 +19,7 @@ exclude_lines =
     def __copy__
     def __deepcopy__
     except ImportError:
+    except ModuleNotFoundError:
     if PYPY:
     if TYPE_CHECKING:
     if "\w+" in sys\.modules:

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: patch
+
+If multiple explicit examples (from :func:`@example() <hypothesis.example>`)
+raise a Skip exception, for consistency with generated examples we now re-raise
+the first instead of collecting them into an ExceptionGroup (:issue:`3453`).

hypothesis-python/src/hypothesis/core.py

@@ -437,7 +437,11 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s
                     err = new
 
                 yield (fragments_reported, err)
-                if state.settings.report_multiple_bugs and pytest_shows_exceptiongroups:
+                if (
+                    state.settings.report_multiple_bugs
+                    and pytest_shows_exceptiongroups
+                    and not isinstance(err, skip_exceptions_to_reraise())
+                ):
                     continue
                 break
             finally:
@@ -1192,6 +1196,14 @@ def wrapped_test(*arguments, **kwargs):
                 # If we're not going to report multiple bugs, we would have
                 # stopped running explicit examples at the first failure.
                 assert len(errors) == 1 or state.settings.report_multiple_bugs
+
+                # If an explicit example raised a 'skip' exception, ensure it's never
+                # wrapped up in an exception group.  Because we break out of the loop
+                # immediately on finding a skip, if present it's always the last error.
+                if isinstance(errors[-1][1], skip_exceptions_to_reraise()):
+                    # Covered by `test_issue_3453_regression`, just in a subprocess.
+                    del errors[:-1]  # pragma: no cover
+
                 _raise_to_user(errors, state.settings, [], " in explicit examples")
 
             # If there were any explicit examples, they all ran successfully.

hypothesis-python/src/hypothesis/extra/codemods.py

@@ -197,7 +197,10 @@ def leave_Call(self, original_node, updated_node):
         # Get the actual function object so that we can inspect the signature.
         # This does e.g. incur a dependency on Numpy to fix Numpy-dependent code,
         # but having a single source of truth about the signatures is worth it.
-        params = signature(get_fn(*qualnames)).parameters.values()
+        try:
+            params = signature(get_fn(*qualnames)).parameters.values()
+        except ModuleNotFoundError:
+            return updated_node
 
         # st.floats() has a new allow_subnormal kwonly argument not at the end,
         # so we do a bit more of a dance here.

hypothesis-python/tests/pytest/test_skipping.py

@@ -39,3 +39,25 @@ def test_no_falsifying_example_if_pytest_skip(testdir):
     )
     out = "\n".join(result.stdout.lines)
     assert "Falsifying example" not in out
+
+
+def test_issue_3453_regression(testdir):
+    """If ``pytest.skip() is called during a test, Hypothesis should not
+    continue running the test and shrink process, nor should it print anything
+    about falsifying examples."""
+    script = testdir.makepyfile(
+        """
+from hypothesis import example, given, strategies as st
+import pytest
+
+@given(value=st.none())
+@example("hello")
+@example("goodbye")
+def test_skip_on_first_skipping_example(value):
+    assert value is not None
+    assert value != "hello"  # queue up a non-skip error which must be discarded
+    pytest.skip()
+"""
+    )
+    result = testdir.runpytest(script, "--tb=native")
+    result.assert_outcomes(skipped=1)

requirements/tools.txt

@@ -35,7 +35,7 @@ bleach==5.0.1
     # via readme-renderer
 build==0.8.0
     # via pip-tools
-certifi==2022.6.15.1
+certifi==2022.9.14
     # via requests
 cffi==1.15.1
     # via cryptography
@@ -94,7 +94,7 @@ flake8-2020==1.7.0
     # via -r requirements/tools.in
 flake8-bandit==4.1.1
     # via -r requirements/tools.in
-flake8-bugbear==22.8.23
+flake8-bugbear==22.9.11
     # via -r requirements/tools.in
 flake8-builtins==1.5.3
     # via -r requirements/tools.in
@@ -124,7 +124,7 @@ gitdb==4.0.9
     # via gitpython
 gitpython==3.1.27
     # via bandit
-idna==3.3
+idna==3.4
     # via requests
 imagesize==1.4.1
     # via sphinx
@@ -149,7 +149,7 @@ jeepney==0.8.0
     #   secretstorage
 jinja2==3.1.2
     # via sphinx
-keyring==23.9.1
+keyring==23.9.3
     # via twine
 lark-parser==0.12.0
     # via -r requirements/tools.in
@@ -232,15 +232,15 @@ pygments==2.13.0
     #   sphinx
 pyparsing==3.0.9
     # via packaging
-pyright==1.1.270
+pyright==1.1.271
     # via -r requirements/tools.in
 pytest==7.1.3
     # via -r requirements/tools.in
 python-dateutil==2.8.2
     # via -r requirements/tools.in
 pytz==2022.2.1
     # via babel
-pyupgrade==2.37.3
+pyupgrade==2.38.0
     # via shed
 pyyaml==6.0
     # via
@@ -264,7 +264,7 @@ rich==12.5.1
     # via twine
 secretstorage==3.3.3
     # via keyring
-shed==0.10.2
+shed==0.10.3
     # via -r requirements/tools.in
 six==1.16.0
     # via
@@ -288,7 +288,7 @@ sphinx==5.1.1
     #   sphinx-codeautolink
     #   sphinx-hoverxref
     #   sphinx-rtd-theme
-sphinx-codeautolink==0.11.0
+sphinx-codeautolink==0.12.0
     # via -r requirements/tools.in
 sphinx-hoverxref==1.1.3
     # via -r requirements/tools.in
@@ -330,7 +330,7 @@ tomli==2.0.1
     #   tox
 tox==3.26.0
     # via -r requirements/tools.in
-traitlets==5.3.0
+traitlets==5.4.0
     # via
     #   ipython
     #   matplotlib-inline

tooling/src/hypothesistooling/__main__.py

@@ -380,9 +380,9 @@ def run_tox(task, version, *args):
 # When a version is added or removed, manually update the env lists in tox.ini and
 # workflows/main.yml, and the `Programming Language ::` specifiers in setup.py
 PYTHONS = {
-    "3.7": "3.7.13",
-    "3.8": "3.8.13",
-    "3.9": "3.9.13",
+    "3.7": "3.7.14",
+    "3.8": "3.8.14",
+    "3.9": "3.9.14",
     "3.10": "3.10.7",
     "3.11": "3.11-dev",
     "3.12": "3.12-dev",