add metadata.spans

Author: tybug

hypothesis-python/RELEASE.rst

@@ -1,5 +1,5 @@
 RELEASE_TYPE: patch
 
-This release adds the experimental and unstable |OBSERVABILITY_CHOICE_NODES| option for :ref:`observability <observability>`, which includes the choice sequence in ``metadata.choice_nodes`` for test case observations if set.
+This release adds the experimental and unstable |OBSERVABILITY_CHOICES| option for :ref:`observability <observability>`, which includes the choice sequence in ``metadata.choice_nodes`` for test case observations if set.
 
-We are actively working towards a better interface for this. Feel free to use |OBSERVABILITY_CHOICE_NODES| to experiment, but don't rely on it yet!
+We are actively working towards a better interface for this. Feel free to use |OBSERVABILITY_CHOICES| to experiment, but don't rely on it yet!

hypothesis-python/docs/prolog.rst

@@ -120,10 +120,12 @@
 .. |PrimitiveProvider.observe_information_messages| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.observe_information_messages`
 .. |PrimitiveProvider.per_test_case_context_manager| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.per_test_case_context_manager`
 .. |PrimitiveProvider.add_observability_callback| replace:: :data:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.add_observability_callback`
+.. |PrimitiveProvider.span_start| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.span_start`
+.. |PrimitiveProvider.span_end| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.span_end`
 
 .. |AVAILABLE_PROVIDERS| replace:: :data:`~hypothesis.internal.conjecture.providers.AVAILABLE_PROVIDERS`
 .. |TESTCASE_CALLBACKS| replace:: :data:`~hypothesis.internal.observability.TESTCASE_CALLBACKS`
-.. |OBSERVABILITY_CHOICE_NODES| replace:: :data:`~hypothesis.internal.observability.OBSERVABILITY_CHOICE_NODES`
+.. |OBSERVABILITY_CHOICES| replace:: :data:`~hypothesis.internal.observability.OBSERVABILITY_CHOICES`
 .. |BUFFER_SIZE| replace:: :data:`~hypothesis.internal.conjecture.engine.BUFFER_SIZE`
 .. |MAX_SHRINKS| replace:: :data:`~hypothesis.internal.conjecture.engine.MAX_SHRINKS`
 .. |MAX_SHRINKING_SECONDS| replace:: :data:`~hypothesis.internal.conjecture.engine.MAX_SHRINKING_SECONDS`

hypothesis-python/docs/reference/internals.rst

@@ -32,7 +32,7 @@ Observability
 
 .. autodata:: hypothesis.internal.observability.TESTCASE_CALLBACKS
 .. autodata:: hypothesis.internal.observability.OBSERVABILITY_COLLECT_COVERAGE
-.. autodata:: hypothesis.internal.observability.OBSERVABILITY_CHOICE_NODES
+.. autodata:: hypothesis.internal.observability.OBSERVABILITY_CHOICES
 
 Engine constants
 ----------------

hypothesis-python/docs/reference/schema_metadata.json

@@ -59,7 +59,7 @@
         },
         "choice_nodes": {
             "type": ["array", "null"],
-            "description": ".. warning::\n\n  EXPERIMENTAL AND UNSTABLE. This attribute may change or disappear without warning.\n\nThe sequence of choices made during this test case. This includes the choice value, as well as its constraints and whether it was forced or not. The choice sequence is a relatively low-level implementation detail of Hypothesis, and is exposed here for users building tools or research on top of Hypothesis. See |PrimitiveProvider| for more details about the choice sequence.\n\nOnly present if |OBSERVABILITY_CHOICE_NODES| is ``True``.",
+            "description": ".. warning::\n\n  EXPERIMENTAL AND UNSTABLE. This attribute may change format or disappear without warning.\n\nThe sequence of choices made during this test case. This includes the choice value, as well as its constraints and whether it was forced or not.\n\nOnly present if |OBSERVABILITY_CHOICES| is ``True``.\n\n.. note::\n\n  The choice sequence is a relatively low-level implementation detail of Hypothesis, and is exposed in observability for users building tools or research on top of Hypothesis. See |PrimitiveProvider| for more details about the choice sequence.",
             "items": {
                 "type": "object",
                 "properties": {
@@ -77,12 +77,17 @@
                     },
                     "was_forced": {
                         "type": "boolean",
-                        "description": "Whether this choice was forced. As an implementation detail, Hypothesis occasionally requires that some choices take on a specific value, for instance to end generation of collection elements early for performance. These values are \"forced\", and have ``was_forced = True``."
+                        "description": "Whether this choice was forced. As an implementation detail, Hypothesis occasionally requires that some choices take on a specific value, for instance to end generation of collection elements early for performance. These values are called \"forced\", and have ``was_forced = True``."
                     }
                 },
                 "required": ["type", "value", "constraints", "was_forced"],
                 "additionalProperties": false
             }
+        },
+        "spans": {
+            "type": "array",
+            "items": {"type": "array"},
+            "description": ".. warning::\n\n  EXPERIMENTAL AND UNSTABLE. This attribute may change format or disappear without warning.\n\nThe semantically-meaningful spans of the choice sequence of this test case.\n\nEach span has the format ``[label, start, end, discarded]``, where:\n\n* ``label`` is an opaque integer-value string shared by all spans drawn from a particular strategy.\n* ``start`` and ``end`` are indices into the choice sequence for this span, such that ``choices[start:end]`` are the corresponding choices.\n* ``discarded`` is a boolean indicating whether this span was discarded (see |PrimitiveProvider.span_end|).\n\nOnly present if |OBSERVABILITY_CHOICES| is ``True``.\n\n.. note::\n\n  Spans are a relatively low-level implementation detail of Hypothesis, and are exposed in observability for users building tools or research on top of Hypothesis. See |PrimitiveProvider| (and particularly |PrimitiveProvider.span_start| and |PrimitiveProvider.span_end|) for more details about spans."
         }
     },
     "required": ["traceback", "reproduction_decorator", "predicates", "backend", "sys_argv", "os_getpid", "imported_at", "data_status", "interesting_origin", "choice_nodes"],

hypothesis-python/src/hypothesis/internal/observability.py

@@ -45,7 +45,7 @@
 if TYPE_CHECKING:
     from typing import TypeAlias
 
-    from hypothesis.internal.conjecture.data import ConjectureData, Status
+    from hypothesis.internal.conjecture.data import ConjectureData, Status, Spans
 
 
 @dataclass
@@ -159,6 +159,7 @@ class ObservationMetadata:
     data_status: "Status"
     interesting_origin: Optional[InterestingOrigin]
     choice_nodes: Optional[tuple[ChoiceNode, ...]]
+    spans: Optional["Spans"]
 
     def to_json(self) -> dict[str, Any]:
         data = {
@@ -174,6 +175,24 @@ def to_json(self) -> dict[str, Any]:
             "choice_nodes": (
                 None if self.choice_nodes is None else nodes_to_json(self.choice_nodes)
             ),
+            "spans": (
+                None
+                if self.spans is None
+                else [
+                    (
+                        # span.label is an int, but cast to string to avoid conversion
+                        # to float  (and loss of precision) for large label values.
+                        #
+                        # The value of this label is opaque to consumers anyway, so its
+                        # type shouldn't matter as long as it's consistent.
+                        str(span.label),
+                        span.start,
+                        span.end,
+                        span.discarded,
+                    )
+                    for span in self.spans
+                ]
+            ),
         }
         # check that we didn't forget one
         assert len(data) == len(dataclasses.fields(self))
@@ -315,7 +334,8 @@ def make_testcase(
                 "backend": backend_metadata or {},
                 "data_status": data.status,
                 "interesting_origin": data.interesting_origin,
-                "choice_nodes": data.nodes if OBSERVABILITY_CHOICE_NODES else None,
+                "choice_nodes": data.nodes if OBSERVABILITY_CHOICES else None,
+                "spans": data.spans if OBSERVABILITY_CHOICES else None,
                 **_system_metadata(),
                 # unpack last so it takes precedence for duplicate keys
                 **(metadata or {}),
@@ -360,21 +380,20 @@ def _system_metadata() -> dict[str, Any]:
 OBSERVABILITY_COLLECT_COVERAGE = (
     "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_NOCOVER" not in os.environ
 )
-#: If ``True``, include the ``metadata.choice_nodes`` key in test case
-#: observations.
+#: If ``True``, include the ``metadata.choice_nodes`` and ``metadata.spans`` keys
+#: in test case observations.
 #:
-#: ``False`` by default. ``metadata.choice_nodes`` can be substantial amount of
-#: data, and so must be opted-in to, even when observability is enabled.
+#: ``False`` by default. ``metadata.choice_nodes`` and ``metadata.spans`` can be
+#: a substantial amount of data, and so must be opted-in to, even when
+#: observability is enabled.
 #:
 #: .. warning::
 #:
 #:     EXPERIMENTAL AND UNSTABLE. We are actively working towards a better
 #:     interface for this as of June 2025, and this attribute may disappear or
 #:     be renamed without notice.
 #:
-OBSERVABILITY_CHOICE_NODES = (
-    "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_CHOICE_NODES" in os.environ
-)
+OBSERVABILITY_CHOICES = "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_CHOICES" in os.environ
 
 if OBSERVABILITY_COLLECT_COVERAGE is False and (
     sys.version_info[:2] >= (3, 12)

hypothesis-python/tests/common/utils.py

@@ -20,6 +20,7 @@
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.floats import next_down
 from hypothesis.internal.observability import TESTCASE_CALLBACKS, Observation
+from hypothesis.internal import observability
 from hypothesis.internal.reflection import proxies
 from hypothesis.reporting import default, with_reporter
 from hypothesis.strategies._internal.core import from_type, register_type_strategy
@@ -235,13 +236,18 @@ def raises_warning(expected_warning, match=None):
 
 
 @contextlib.contextmanager
-def capture_observations():
+def capture_observations(*, choices=None):
     ls: list[Observation] = []
     TESTCASE_CALLBACKS.append(ls.append)
+    if choices is not None:
+        observability.OBSERVABILITY_CHOICES = choices
+
     try:
         yield ls
     finally:
         TESTCASE_CALLBACKS.remove(ls.append)
+        if choices is not None:
+            observability.OBSERVABILITY_CHOICES = choices
 
 
 # Specifies whether we can represent subnormal floating point numbers.

hypothesis-python/tests/cover/test_observability.py

@@ -30,6 +30,7 @@
 from hypothesis.database import InMemoryExampleDatabase
 from hypothesis.internal.compat import PYPY
 from hypothesis.internal.conjecture.choice import ChoiceNode, choices_key
+from hypothesis.internal.conjecture.data import Span
 from hypothesis.internal.coverage import IN_COVERAGE_TESTS
 from hypothesis.internal.floats import SIGNALING_NAN, float_to_int, int_to_float
 from hypothesis.internal.intervalsets import IntervalSet
@@ -503,7 +504,7 @@ def test_metadata_to_json():
     def f(n):
         pass
 
-    with capture_observations() as observations:
+    with capture_observations(choices=True) as observations:
         f()
 
     observations = [obs for obs in observations if obs.type == "test_case"]
@@ -519,4 +520,11 @@ def f(n):
             "data_status",
             "interesting_origin",
             "choice_nodes",
+            "spans",
         }
+        assert observation.metadata.choice_nodes is not None
+
+        for span in observation.metadata.spans:
+            assert isinstance(span, Span)
+            assert 0 <= span.start <= len(observation.metadata.choice_nodes)
+            assert 0 <= span.end <= len(observation.metadata.choice_nodes)