Observability from backends

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,6 @@
+RELEASE_TYPE: minor
+
+:ref:`alternative-backends` can now implement a ``.get_observability_data()``
+method, providing the ``x['metadata']['backend']`` dictionary in our
+:doc:`observability output <observability>` (:issue:`3845` and `hypothesis-crosshair#22
+<https://github.com/pschanely/hypothesis-crosshair/issues/22>`__).

hypothesis-python/src/hypothesis/core.py

@@ -1122,6 +1122,7 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None:
                     data._observability_args
                 )
                 self._string_repr = data.provider.realize(self._string_repr)
+                backend_metadata = data.provider.get_observability_data()
                 tc = make_testcase(
                     start_timestamp=self._start_timestamp,
                     test_name_or_nodeid=self.test_identifier,
@@ -1132,6 +1133,7 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None:
                     timing=self._timing_features,
                     coverage=tractable_coverage_report(trace) or None,
                     phase=phase,
+                    backend_metadata=backend_metadata,
                 )
                 deliver_json_blob(tc)
             self._timing_features = {}

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -1240,9 +1240,16 @@ def realize(self, value: T) -> T:
 
         The returned value should be non-symbolic.
         """
-
         return value
 
+    def get_observability_data(self) -> Dict[str, Any]:
+        """Called at the end of the test case when observability mode is active.
+
+        The return value should be a non-symbolic json-encodable dictionary,
+        and will be included as `observation["metadata"]["backend"]`.
+        """
+        return {}
+
     @abc.abstractmethod
     def draw_boolean(
         self,

hypothesis-python/src/hypothesis/internal/observability.py

@@ -17,7 +17,7 @@
 import warnings
 from datetime import date, timedelta
 from functools import lru_cache
-from typing import Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 from hypothesis.configuration import storage_directory
 from hypothesis.errors import HypothesisWarning
@@ -42,6 +42,7 @@ def make_testcase(
     timing: Dict[str, float],
     coverage: Optional[Dict[str, List[int]]] = None,
     phase: Optional[str] = None,
+    backend_metadata: Optional[Dict[str, Any]] = None,
 ) -> dict:
     if data.interesting_origin:
         status_reason = str(data.interesting_origin)
@@ -74,6 +75,7 @@ def make_testcase(
         "metadata": {
             "traceback": getattr(data.extra_information, "_expected_traceback", None),
             "predicates": data._observability_predicates,
+            "backend": backend_metadata or {},
             **_system_metadata(),
         },
         "coverage": coverage,