report all generated arguments in observability

Author: tybug

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This patch fixes a bug where only interactively-generated values (via ``data.draw``) would be reported in the ``arguments`` field of our :doc:`observability output <observability>`. Now, all values are reported.

hypothesis-python/src/hypothesis/internal/conjecture/data.py

@@ -2485,6 +2485,9 @@ def draw(
         label: Optional[int] = None,
         observe_as: Optional[str] = None,
     ) -> "Ex":
+        from hypothesis.internal.observability import TESTCASE_CALLBACKS
+        from hypothesis.strategies._internal.utils import to_jsonable
+
         if self.is_find and not strategy.supports_find:
             raise InvalidArgument(
                 f"Cannot use strategy {strategy!r} within a call to find "
@@ -2521,7 +2524,7 @@ def draw(
             try:
                 strategy.validate()
                 try:
-                    return strategy.do_draw(self)
+                    v = strategy.do_draw(self)
                 finally:
                     # Subtract the time spent in GC to avoid overcounting, as it is
                     # accounted for at the overall example level.
@@ -2530,6 +2533,9 @@ def draw(
             except Exception as err:
                 add_note(err, f"while generating {key[9:]!r} from {strategy!r}")
                 raise
+            if TESTCASE_CALLBACKS:
+                self._observability_args[key] = to_jsonable(v)
+            return v
         finally:
             self.stop_example()
 

hypothesis-python/src/hypothesis/strategies/_internal/core.py

@@ -85,7 +85,6 @@
 )
 from hypothesis.internal.entropy import get_seeder_and_restorer
 from hypothesis.internal.floats import float_of
-from hypothesis.internal.observability import TESTCASE_CALLBACKS
 from hypothesis.internal.reflection import (
     define_function_signature,
     get_pretty_function_description,
@@ -139,11 +138,7 @@
     TextStrategy,
     _check_is_single_character,
 )
-from hypothesis.strategies._internal.utils import (
-    cacheable,
-    defines_strategy,
-    to_jsonable,
-)
+from hypothesis.strategies._internal.utils import cacheable, defines_strategy
 from hypothesis.utils.conventions import not_set
 from hypothesis.vendor.pretty import RepresentationPrinter
 
@@ -2119,15 +2114,13 @@ def draw(self, strategy: SearchStrategy[Ex], label: Any = None) -> Ex:
         check_strategy(strategy, "strategy")
         self.count += 1
         printer = RepresentationPrinter(context=current_build_context())
-        desc = f"Draw {self.count}{'' if label is None else f' ({label})'}: "
+        desc = f"Draw {self.count}{'' if label is None else f' ({label})'}"
         with deprecate_random_in_strategy("{}from {!r}", desc, strategy):
             result = self.conjecture_data.draw(strategy, observe_as=f"generate:{desc}")
-        if TESTCASE_CALLBACKS:
-            self.conjecture_data._observability_args[desc] = to_jsonable(result)
 
         # optimization to avoid needless printer.pretty
         if should_note():
-            printer.text(desc)
+            printer.text(f"{desc}: ")
             printer.pretty(result)
             note(printer.getvalue())
         return result

hypothesis-python/tests/cover/test_observability.py

@@ -69,6 +69,42 @@ def test_observability():
             )
 
 
+def test_capture_unnamed_arguments():
+    @given(st.integers(), st.floats(), st.data())
+    def f(v1, v2, data):
+        data.draw(st.booleans())
+
+    with capture_observations() as observations:
+        f()
+
+    test_cases = [tc for tc in observations if tc["type"] == "test_case"]
+    for test_case in test_cases:
+        assert list(test_case["arguments"].keys()) == [
+            "generate:v1",
+            "generate:v2",
+            "generate:data",
+            "generate:Draw 1",
+        ], test_case
+
+
+def test_capture_named_arguments():
+    @given(named1=st.integers(), named2=st.floats(), data=st.data())
+    def f(named1, named2, data):
+        data.draw(st.booleans())
+
+    with capture_observations() as observations:
+        f()
+
+    test_cases = [tc for tc in observations if tc["type"] == "test_case"]
+    for test_case in test_cases:
+        assert list(test_case["arguments"].keys()) == [
+            "generate:named1",
+            "generate:named2",
+            "generate:data",
+            "generate:Draw 1",
+        ], test_case
+
+
 def test_assume_has_status_reason():
     @given(st.booleans())
     def f(b):