Merge pull request #4083 from Zac-HD/observe-backends

Allow alternative backends to provide observability metadata

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,7 @@
+RELEASE_TYPE: minor
+
+:ref:`alternative-backends` can now implement ``.observe_test_case()``
+and ``observe_information_message()`` methods, to record backend-specific
+metadata and messages in our :doc:`observability output <observability>`
+(:issue:`3845` and `hypothesis-crosshair#22
+<https://github.com/pschanely/hypothesis-crosshair/issues/22>`__).

hypothesis-python/docs/schema_observations.json

@@ -74,16 +74,16 @@
             "type": "object",
             "properties": {
                 "type": {
-                    "enum": [ "info", "alert", "error"],
+                    "enum": ["info", "alert", "error"],
                     "description": "A tag which labels this observation as general information to show the user.  Hypothesis uses info messages to report statistics; alert or error messages can be provided by plugins."
                 },
                 "title": {
                     "type": "string",
                     "description": "The title of this message"
                 },
                 "content": {
-                    "type": "string",
-                    "description": "The body of the message.  May use markdown."
+                    "type": ["string", "object"],
+                    "description": "The body of the message.  Strings are presumed to be human-readable messages in markdown format; dictionaries may contain arbitrary information (as for test-case metadata)."
                 },
                 "property": {
                     "type": "string",
@@ -94,7 +94,7 @@
                     "description": "unix timestamp at which we started running this test function, so that later analysis can group test cases by run."
                 }
             },
-            "required": [ "type", "title", "content", "property", "run_start"],
+            "required": ["type", "title", "content", "property", "run_start"],
             "additionalProperties": false
         }
     ]

hypothesis-python/scripts/other-tests.sh

@@ -67,7 +67,7 @@ if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel ==
   $PYTEST tests/ghostwriter/
   pip uninstall -y black
 
-  if [ "$(python -c "import platform; print(platform.python_implementation() not in {'PyPy', 'GraalVM'})")" = "True" ] ; then
+  if [ "$HYPOTHESIS_PROFILE" != "crosshair" ] && [ "$(python -c "import platform; print(platform.python_implementation() not in {'PyPy', 'GraalVM'})")" = "True" ] ; then
     $PYTEST tests/array_api tests/numpy
   fi
 fi

hypothesis-python/src/hypothesis/core.py

@@ -77,7 +77,11 @@
     get_type_hints,
     int_from_bytes,
 )
-from hypothesis.internal.conjecture.data import ConjectureData, Status
+from hypothesis.internal.conjecture.data import (
+    ConjectureData,
+    PrimitiveProvider,
+    Status,
+)
 from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner
 from hypothesis.internal.conjecture.junkdrawer import (
     ensure_free_stackframes,
@@ -1132,10 +1136,28 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None:
                     timing=self._timing_features,
                     coverage=tractable_coverage_report(trace) or None,
                     phase=phase,
+                    backend_metadata=data.provider.observe_test_case(),
                 )
                 deliver_json_blob(tc)
+                for msg in data.provider.observe_information_messages(
+                    lifetime="test_case"
+                ):
+                    self._deliver_information_message(**msg)
             self._timing_features = {}
 
+    def _deliver_information_message(
+        self, *, type: str, title: str, content: Union[str, dict]
+    ) -> None:
+        deliver_json_blob(
+            {
+                "type": type,
+                "run_start": self._start_timestamp,
+                "property": self.test_identifier,
+                "title": title,
+                "content": content,
+            }
+        )
+
     def run_engine(self):
         """Run the test function many times, on database input and generated
         input, using the Conjecture engine.
@@ -1160,15 +1182,16 @@ def run_engine(self):
         # on different inputs.
         runner.run()
         note_statistics(runner.statistics)
-        deliver_json_blob(
-            {
-                "type": "info",
-                "run_start": self._start_timestamp,
-                "property": self.test_identifier,
-                "title": "Hypothesis Statistics",
-                "content": describe_statistics(runner.statistics),
-            }
-        )
+        if TESTCASE_CALLBACKS:
+            self._deliver_information_message(
+                type="info",
+                title="Hypothesis Statistics",
+                content=describe_statistics(runner.statistics),
+            )
+            for msg in (
+                p if isinstance(p := runner.provider, PrimitiveProvider) else p(None)
+            ).observe_information_messages(lifetime="test_function"):
+                self._deliver_information_message(**msg)
 
         if runner.call_count == 0:
             return

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -1186,6 +1186,14 @@ def as_result(self) -> "ConjectureResult":
 BYTE_MASKS = [(1 << n) - 1 for n in range(8)]
 BYTE_MASKS[0] = 255
 
+_Lifetime: TypeAlias = Literal["test_case", "test_function"]
+
+
+class _BackendInfoMsg(TypedDict):
+    type: str
+    title: str
+    content: Union[str, Dict[str, Any]]
+
 
 class PrimitiveProvider(abc.ABC):
     # This is the low-level interface which would also be implemented
@@ -1212,7 +1220,7 @@ class PrimitiveProvider(abc.ABC):
     # lifetime can access the passed ConjectureData object.
     #
     # Non-hypothesis providers probably want to set a lifetime of test_function.
-    lifetime = "test_function"
+    lifetime: _Lifetime = "test_function"
 
     # Solver-based backends such as hypothesis-crosshair use symbolic values
     # which record operations performed on them in order to discover new paths.
@@ -1240,9 +1248,28 @@ def realize(self, value: T) -> T:
 
         The returned value should be non-symbolic.
         """
-
         return value
 
+    def observe_test_case(self) -> Dict[str, Any]:
+        """Called at the end of the test case when observability mode is active.
+
+        The return value should be a non-symbolic json-encodable dictionary,
+        and will be included as `observation["metadata"]["backend"]`.
+        """
+        return {}
+
+    def observe_information_messages(
+        self, *, lifetime: _Lifetime
+    ) -> Iterable[_BackendInfoMsg]:
+        """Called at the end of each test case and again at end of the test function.
+
+        Return an iterable of `{type: info/alert/error, title: str, content: str|dict}`
+        dictionaries to be delivered as individual information messages.
+        (Hypothesis adds the `run_start` timestamp and `property` name for you.)
+        """
+        assert lifetime in ("test_case", "test_function")
+        yield from []
+
     @abc.abstractmethod
     def draw_boolean(
         self,
@@ -1307,7 +1334,6 @@ class HypothesisProvider(PrimitiveProvider):
     lifetime = "test_case"
 
     def __init__(self, conjecturedata: Optional["ConjectureData"], /):
-        assert conjecturedata is not None
         super().__init__(conjecturedata)
 
     def draw_boolean(

hypothesis-python/src/hypothesis/internal/observability.py

@@ -17,7 +17,7 @@
 import warnings
 from datetime import date, timedelta
 from functools import lru_cache
-from typing import Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 from hypothesis.configuration import storage_directory
 from hypothesis.errors import HypothesisWarning
@@ -42,6 +42,7 @@ def make_testcase(
     timing: Dict[str, float],
     coverage: Optional[Dict[str, List[int]]] = None,
     phase: Optional[str] = None,
+    backend_metadata: Optional[Dict[str, Any]] = None,
 ) -> dict:
     if data.interesting_origin:
         status_reason = str(data.interesting_origin)
@@ -74,6 +75,7 @@ def make_testcase(
         "metadata": {
             "traceback": getattr(data.extra_information, "_expected_traceback", None),
             "predicates": data._observability_predicates,
+            "backend": backend_metadata or {},
             **_system_metadata(),
         },
         "coverage": coverage,

hypothesis-python/tests/common/utils.py

@@ -18,6 +18,7 @@
 from hypothesis.errors import HypothesisDeprecationWarning
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.floats import next_down
+from hypothesis.internal.observability import TESTCASE_CALLBACKS
 from hypothesis.internal.reflection import proxies
 from hypothesis.reporting import default, with_reporter
 from hypothesis.strategies._internal.core import from_type, register_type_strategy
@@ -232,6 +233,16 @@ def raises_warning(expected_warning, match=None):
             yield r
 
 
+@contextlib.contextmanager
+def capture_observations():
+    ls = []
+    TESTCASE_CALLBACKS.append(ls.append)
+    try:
+        yield ls
+    finally:
+        TESTCASE_CALLBACKS.remove(ls.append)
+
+
 # Specifies whether we can represent subnormal floating point numbers.
 # IEE-754 requires subnormal support, but it's often disabled anyway by unsafe
 # compiler options like `-ffast-math`.  On most hardware that's even a global

hypothesis-python/tests/conjecture/test_alt_backend.py

@@ -10,9 +10,10 @@
 
 import math
 import sys
+from collections.abc import Sequence
 from contextlib import contextmanager
 from random import Random
-from typing import Optional, Sequence
+from typing import Optional
 
 import pytest
 
@@ -31,6 +32,7 @@
 from hypothesis.internal.intervalsets import IntervalSet
 
 from tests.common.debug import minimal
+from tests.common.utils import capture_observations
 from tests.conjecture.common import ir_nodes
 
 
@@ -358,7 +360,7 @@ def test_function(n):
 
 
 def test_flaky_with_backend():
-    with temp_register_backend("trivial", TrivialProvider):
+    with temp_register_backend("trivial", TrivialProvider), capture_observations():
 
         calls = 0
 
@@ -428,3 +430,42 @@ def test_function(data):
             assert n1 <= n2
 
         test_function()
+
+
+class ObservableProvider(TrivialProvider):
+    def observe_test_case(self):
+        return {"msg_key": "some message", "data_key": [1, "2", {}]}
+
+    def observe_information_messages(self, *, lifetime):
+        if lifetime == "test_case":
+            yield {"type": "info", "title": "trivial-data", "content": {"k2": "v2"}}
+        else:
+            assert lifetime == "test_function"
+            yield {"type": "alert", "title": "Trivial alert", "content": "message here"}
+            yield {"type": "info", "title": "trivial-data", "content": {"k2": "v2"}}
+
+
+def test_custom_observations_from_backend():
+    with (
+        temp_register_backend("observable", ObservableProvider),
+        capture_observations() as ls,
+    ):
+
+        @given(st.none())
+        @settings(backend="observable", database=None)
+        def test_function(_):
+            pass
+
+        test_function()
+
+    assert len(ls) >= 3
+    cases = [t["metadata"]["backend"] for t in ls if t["type"] == "test_case"]
+    assert {"msg_key": "some message", "data_key": [1, "2", {}]} in cases
+
+    infos = [
+        {k: v for k, v in t.items() if k in ("title", "content")}
+        for t in ls
+        if t["type"] != "test_case"
+    ]
+    assert {"title": "Trivial alert", "content": "message here"} in infos
+    assert {"title": "trivial-data", "content": {"k2": "v2"}} in infos

hypothesis-python/tests/cover/test_observability.py

@@ -8,8 +8,6 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
-import contextlib
-
 import pytest
 
 from hypothesis import (
@@ -23,23 +21,14 @@
     target,
 )
 from hypothesis.database import InMemoryExampleDatabase
-from hypothesis.internal.observability import TESTCASE_CALLBACKS
 from hypothesis.stateful import (
     RuleBasedStateMachine,
     invariant,
     rule,
     run_state_machine_as_test,
 )
 
-
-@contextlib.contextmanager
-def capture_observations():
-    ls = []
-    TESTCASE_CALLBACKS.append(ls.append)
-    try:
-        yield ls
-    finally:
-        TESTCASE_CALLBACKS.remove(ls.append)
+from tests.common.utils import capture_observations
 
 
 @seed("deterministic so we don't miss some combination of features")