Merge pull request #4011 from jobh/reinstate-coverage

Reinstate coverage

Author: jobh

hypothesis-python/.coveragerc

@@ -1,5 +1,7 @@
 [run]
+parallel = True
 branch = True
+source = hypothesis
 omit =
     **/_hypothesis_ftz_detector.py
     **/_hypothesis_pytestplugin.py

hypothesis-python/RELEASE.rst

@@ -0,0 +1,6 @@
+RELEASE_TYPE: patch
+
+Fixes and reinstates full coverage of internal tests, which was accidentally
+disabled in :pull:`3935`.
+
+Closes :issue:`4003`.

hypothesis-python/docs/strategies.rst

@@ -212,6 +212,26 @@ loading our pytest plugin from your ``conftest.py`` instead::
     echo "pytest_plugins = ['hypothesis.extra.pytestplugin']\n" > tests/conftest.py
     pytest -p "no:hypothesispytest" ...
 
+Another alternative, which we in fact use in our CI self-tests because it works
+well also with parallel tests, is to automatically start coverage early for all
+new processes if an environment variable is set.
+This automatic starting is set up by the PyPi package :pypi:`coverage_enable_subprocess`.
+
+This means all configuration must be done in ``.coveragerc``, and not on the
+command line::
+
+    [run]
+    parallel = True
+    source = ...
+
+Then, set the relevant environment variable and run normally::
+
+    python -m pip install coverage_enable_subprocess
+    export COVERAGE_PROCESS_START=$PATH/.coveragerc
+    pytest [-n auto] ...
+    coverage combine
+    coverage report
+
 
 .. _alternative-backends:
 

hypothesis-python/src/hypothesis/core.py

@@ -1104,7 +1104,9 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None:
             if TESTCASE_CALLBACKS:
                 if runner := getattr(self, "_runner", None):
                     phase = runner._current_phase
-                elif self.failed_normally or self.failed_due_to_deadline:
+                elif (
+                    self.failed_normally or self.failed_due_to_deadline
+                ):  # pragma: no cover  # FIXME
                     phase = "shrink"
                 else:  # pragma: no cover  # in case of messing with internals
                     phase = "unknown"

hypothesis-python/src/hypothesis/extra/_patching.py

@@ -121,7 +121,7 @@ def __call_node_to_example_dec(self, node, via):
                     cst.Module([]).code_for_node(via),
                     mode=black.FileMode(line_length=self.line_length),
                 )
-            except (ImportError, AttributeError):
+            except (ImportError, AttributeError):  # pragma: no cover
                 return None  # See https://github.com/psf/black/pull/4224
             via = cst.parse_expression(pretty.strip())
         return cst.Decorator(via)

hypothesis-python/src/hypothesis/internal/entropy.py

@@ -127,7 +127,7 @@ def my_WORKING_hook():
                     "PRNG. See the docs for `register_random` for more "
                     "details."
                 )
-            elif not FREE_THREADED_CPYTHON:
+            elif not FREE_THREADED_CPYTHON:  # pragma: no cover  # FIXME
                 # On CPython, check for the free-threaded build because
                 # gc.get_referrers() ignores objects with immortal refcounts
                 # and objects are immortalized in the Python 3.13

hypothesis-python/src/hypothesis/stateful.py

@@ -388,7 +388,7 @@ def _add_result_to_targets(self, targets, result):
         for target in targets:
             name = self._new_name(target)
 
-            def printer(obj, p, cycle, name=name):
+            def printer(obj, p, cycle, name=name):  # pragma: no cover  # FIXME
                 return p.text(name)
 
             self.__printer.singleton_pprinters.setdefault(id(result), printer)

hypothesis-python/src/hypothesis/statistics.py

@@ -48,7 +48,7 @@ def format_ms(times):
     """
     ordered = sorted(times)
     n = len(ordered) - 1
-    if n < 0 or any(math.isnan(t) for t in ordered):
+    if n < 0 or any(math.isnan(t) for t in ordered):  # pragma: no cover
         return "NaN ms"
     lower = int(ordered[int(math.floor(n * 0.05))] * 1000)
     upper = int(ordered[int(math.ceil(n * 0.95))] * 1000)

hypothesis-python/tests/cover/test_composite.py

@@ -196,19 +196,22 @@ def test_drawfn_cannot_be_instantiated():
 
 
 @pytest.mark.skipif(sys.version_info[:2] == (3, 9), reason="stack depth varies???")
-@pytest.mark.skipif(sys.version_info[:2] <= (3, 11), reason="TEMP: see PR #3961")
 def test_warns_on_strategy_annotation():
     # TODO: print the stack on Python 3.10 and 3.11 to determine the appropriate
     #       stack depth to use.  Consider adding a debug-print if IN_COVERAGE_TESTS
     #       and the relevant depth is_hypothesis_file(), for easier future fixing.
+    #
+    # Meanwhile, the test is not skipped on 3.10/3.11 as it is still required for
+    # coverage of the warning-generating branch.
     with pytest.warns(HypothesisWarning, match="Return-type annotation") as w:
 
         @st.composite
         def my_integers(draw: st.DrawFn) -> st.SearchStrategy[int]:
             return draw(st.integers())
 
-    assert len(w.list) == 1
-    assert w.list[0].filename == __file__  # check stacklevel points to user code
+    if sys.version_info[:2] > (3, 11):  # TEMP: see PR #3961
+        assert len(w.list) == 1
+        assert w.list[0].filename == __file__  # check stacklevel points to user code
 
 
 def test_composite_allows_overload_without_draw():

hypothesis-python/tox.ini

@@ -218,20 +218,29 @@ allowlist_externals =
 setenv=
     PYTHONWARNDEFAULTENCODING=1
     HYPOTHESIS_INTERNAL_COVERAGE=true
+    COVERAGE_PROCESS_START=.coveragerc
 commands_pre =
     rm -f branch-check*
     pip install .[zoneinfo]
+    pip install coverage_enable_subprocess
+commands_post =
+    pip uninstall -y coverage_enable_subprocess
 # Produce a coverage report even if the test suite fails.
 # (The tox task will still count as failed.)
 ignore_errors = true
+# We've had problems correctly measuring coverage using pytest-cov when running
+# in parallel, so instead we start coverage implicitly on all (sub-)processes by
+# way of the coverage_enable_subprocesses installation. This requires all options
+# to be set in .coveragerc (including source), but that's ok as it is overridden
+# by --cov=... in conjecture-coverage.
 commands =
     python -bb -X dev -m pytest -n auto --ff {posargs} \
-        --cov=hypothesis.internal.conjecture --cov-config=.coveragerc \
         tests/cover tests/conjecture tests/datetime tests/numpy tests/pandas tests/lark \
         tests/redis tests/dpcontracts tests/codemods tests/typing_extensions tests/patching tests/test_annotated_types.py
+    python -m coverage combine
+    python -m coverage report
     python scripts/validate_branch_check.py
 
-
 [testenv:conjecture-coverage]
 deps =
     -r../requirements/coverage.txt