document hypothesis observations metadata

tybug · tybug · commit a02dbacebac7 · 2025-06-03T20:48:28.000-04:00
diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst
@@ -0,0 +1,3 @@
+RELEASE_TYPE: minor
+
+This release adds |OBSERVABILITY_CHOICE_NODES|, a new option for :ref:`observability <observability>`, which includes the choice sequence in ``metadata.choice_nodes`` for test case observations if set.
diff --git a/hypothesis-python/docs/prolog.rst b/hypothesis-python/docs/prolog.rst
@@ -116,17 +116,21 @@
 .. |PrimitiveProvider.draw_string| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_string`
 .. |PrimitiveProvider.draw_bytes| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_bytes`
 .. |PrimitiveProvider.on_observation| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.on_observation`
+.. |PrimitiveProvider.observe_test_case| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.observe_test_case`
+.. |PrimitiveProvider.observe_information_messages| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.observe_information_messages`
 .. |PrimitiveProvider.per_test_case_context_manager| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.per_test_case_context_manager`
 .. |PrimitiveProvider.add_observability_callback| replace:: :data:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.add_observability_callback`
 
 .. |AVAILABLE_PROVIDERS| replace:: :data:`~hypothesis.internal.conjecture.providers.AVAILABLE_PROVIDERS`
 .. |TESTCASE_CALLBACKS| replace:: :data:`~hypothesis.internal.observability.TESTCASE_CALLBACKS`
+.. |OBSERVABILITY_CHOICE_NODES| replace:: :data:`~hypothesis.internal.observability.OBSERVABILITY_CHOICE_NODES`
 .. |BUFFER_SIZE| replace:: :data:`~hypothesis.internal.conjecture.engine.BUFFER_SIZE`
 .. |MAX_SHRINKS| replace:: :data:`~hypothesis.internal.conjecture.engine.MAX_SHRINKS`
 .. |MAX_SHRINKING_SECONDS| replace:: :data:`~hypothesis.internal.conjecture.engine.MAX_SHRINKING_SECONDS`
 .. |BackendCannotProceed| replace:: :exc:`~hypothesis.errors.BackendCannotProceed`
 
 .. |@rule| replace:: :func:`@rule <hypothesis.stateful.rule>`
+.. |@precondition| replace:: :func:`@precondition <hypothesis.stateful.precondition>`
 .. |RuleBasedStateMachine| replace:: :class:`~hypothesis.stateful.RuleBasedStateMachine`
 .. |run_state_machine_as_test| replace:: :func:`~hypothesis.stateful.run_state_machine_as_test`
 
diff --git a/hypothesis-python/docs/reference/integrations.rst b/hypothesis-python/docs/reference/integrations.rst
@@ -162,6 +162,15 @@ including Gson in Java, ``JSON.parse()`` in Ruby, and of course in Python.
    :hide_key: /additionalProperties, /type
 
 
+Hypothesis Metadata
+^^^^^^^^^^^^^^^^^^^
+
+While the observability format is agnostic to the property-based testing library which generated it, Hypothesis includes specific values in the ``metadata`` key for test cases. You may rely on these being present if and only if the observation was generated by Hypothesis.
+
+.. jsonschema:: ./schema_metadata.json
+   :hide_key: /additionalProperties, /type
+
+
 .. _pytest-plugin:
 
 The Hypothesis pytest plugin
diff --git a/hypothesis-python/docs/reference/internals.rst b/hypothesis-python/docs/reference/internals.rst
@@ -30,7 +30,7 @@ Observability
 
 .. autodata:: hypothesis.internal.observability.TESTCASE_CALLBACKS
 .. autodata:: hypothesis.internal.observability.OBSERVABILITY_COLLECT_COVERAGE
-
+.. autodata:: hypothesis.internal.observability.OBSERVABILITY_CHOICE_NODES
 
 Engine constants
 ----------------
diff --git a/hypothesis-python/docs/reference/schema_metadata.json b/hypothesis-python/docs/reference/schema_metadata.json
@@ -0,0 +1,91 @@
+{
+    "title": "Hypothesis Metadata",
+    "description": "Hypothesis-specific values included in the ``metadata`` key of observations for test cases.",
+    "type": "object",
+    "properties": {
+        "traceback": {
+            "type": ["string", "null"],
+            "description": "The traceback for failing tests, if and only if ``status == \"failed\"``."
+        },
+        "reproduction_decorator": {
+            "type": ["string", "null"],
+            "description": "The ``@reproduce_failure`` decorator string for failing tests, if and only if ``status == \"failed\"``."
+        },
+        "predicates": {
+            "type": "object",
+            "description": "The number of times each |assume| and |@precondition| predicate was satisfied (``True``) and not satisfied (``False``).",
+            "additionalProperties": {
+                "type": "object",
+                "properties": {
+                    "satisfied": {
+                        "type": "integer",
+                        "minimum": 0,
+                        "description": "The number of times this predicate was satisfied (``True``)."
+                    },
+                    "unsatisfied": {
+                        "type": "integer",
+                        "minimum": 0,
+                        "description": "The number of times this predicate was not satisfied (``False``)."
+                    }
+                },
+                "required": ["satisfied", "unsatisfied"],
+                "additionalProperties": false
+            }
+        },
+        "backend": {
+            "type": "object",
+            "description": "Backend-specific observations from |PrimitiveProvider.observe_test_case| and |PrimitiveProvider.observe_information_messages|."
+        },
+        "sys_argv": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": "The result of ``sys.argv``."
+        },
+        "os_getpid": {
+            "type": "integer",
+            "description": "The result of ``os.getpid()``."
+        },
+        "imported_at": {
+            "type": "number",
+            "description": "The unix timestamp when Hypothesis was imported."
+        },
+        "data_status": {
+            "type": "number",
+            "enum": [0, 1, 2, 3],
+            "description": "The internal status of the ConjectureData for this test case. The values are as follows: ``Status.OVERRUN = 0``, ``Status.INVALID = 1``, ``Status.VALID = 2``, and ``Status.INTERESTING = 3``."
+        },
+        "interesting_origin": {
+            "type": ["string", "null"],
+            "description": "The internal InterestingOrigin object for failing tests, if and only if ``status == \"failed\"``. The ``traceback`` string value is derived from this object."
+        },
+        "choice_nodes": {
+            "type": ["array", "null"],
+            "description": "The sequence of choices made during this test case. This includes the choice value, as well as its constraints and whether it was forced or not. The choice sequence is a relatively low-level implementation detail of Hypothesis, and is exposed here for users building tools or research on top of Hypothesis. See |PrimitiveProvider| for more details about the choice sequence.\n\nOnly present if |OBSERVABILITY_CHOICE_NODES| is set.",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "type": {
+                        "type": "string",
+                        "enum": ["integer", "float", "string", "bytes", "boolean"],
+                        "description": "The type of choice made. Corresponds to a call to |PrimitiveProvider.draw_integer|, |PrimitiveProvider.draw_float|, |PrimitiveProvider.draw_string|, |PrimitiveProvider.draw_bytes|, or |PrimitiveProvider.draw_boolean|."
+                    },
+                    "value": {
+                        "description": "The value of the choice. Corresponds to the value returned by a ``PrimitiveProvider.draw_*`` method.\n\n``NaN`` float values are returned as ``[\"float\", <float64_int_value>]``, to distinguish ``NaN`` floats with nonstandard bit patterns. Integers with  ``abs(value) >= 2**63`` are returned as ``[\"integer\", str(value)]``, for compatibility with tools with integer size limitations. Bytes are returned as ``[\"bytes\", base64.b64encode(value)]``."
+                    },
+                    "constraints": {
+                        "type": "object",
+                        "description": "The constraints for this choice. Corresponds to the constraints passed to a ``PrimitiveProvider.draw_*`` method. ``NaN`` float values, integers with ``abs(value) >= 2**63``, and byte values for constraints are transformed as for the ``value`` attribute."
+                    },
+                    "was_forced": {
+                        "type": "boolean",
+                        "description": "Whether this choice was forced. As an implementation detail, Hypothesis occasionally requires that some choices take on a specific value, for instance to end generation of collection elements early for performance. These values are \"forced\", and have ``was_forced = True``."
+                    }
+                },
+                "required": ["type", "value", "constraints", "was_forced"],
+                "additionalProperties": false
+            }
+        }
+    },
+    "required": ["traceback", "reproduction_decorator", "predicates", "backend", "sys_argv", "os_getpid", "imported_at", "data_status", "interesting_origin", "choice_nodes"],
+    "additionalProperties": false
+}
diff --git a/hypothesis-python/src/hypothesis/internal/observability.py b/hypothesis-python/src/hypothesis/internal/observability.py
@@ -65,8 +65,12 @@ def _choice_to_json(choice: Union[ChoiceT, None]) -> Any:
         return None
     # see the note on the same check in to_jsonable for why we cast large
     # integers to floats.
-    if isinstance(choice, int) and not isinstance(choice, bool) and choice > 2**63:
-        return ["integer", float(choice)]
+    if (
+        isinstance(choice, int)
+        and not isinstance(choice, bool)
+        and abs(choice) >= 2**63
+    ):
+        return ["integer", str(choice)]
     elif isinstance(choice, bytes):
         return ["bytes", base64.b64encode(choice).decode()]
     elif isinstance(choice, float) and math.isnan(choice):
@@ -352,6 +356,11 @@ def _system_metadata() -> dict[str, Any]:
 OBSERVABILITY_COLLECT_COVERAGE = (
     "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_NOCOVER" not in os.environ
 )
+#: If ``True``, include the ``metadata.choice_nodes`` key in test case
+#: observations.
+#:
+#: ``False`` by default. ``metadata.choice_nodes`` can be substantial amount of
+#: data, and so must be opted-in to, even when observability is enabled.
 OBSERVABILITY_CHOICE_NODES = (
     "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_CHOICE_NODES" in os.environ
 )
diff --git a/hypothesis-python/tests/cover/test_observability.py b/hypothesis-python/tests/cover/test_observability.py
@@ -31,7 +31,7 @@
 from hypothesis.internal.compat import PYPY
 from hypothesis.internal.conjecture.choice import ChoiceNode, choices_key
 from hypothesis.internal.coverage import IN_COVERAGE_TESTS
-from hypothesis.internal.floats import SIGNALING_NAN, int_to_float
+from hypothesis.internal.floats import SIGNALING_NAN, float_to_int, int_to_float
 from hypothesis.internal.intervalsets import IntervalSet
 from hypothesis.internal.observability import choices_to_json, nodes_to_json
 from hypothesis.stateful import (
@@ -42,7 +42,7 @@
 )
 
 from tests.common.utils import Why, capture_observations, xfail_on_crosshair
-from tests.conjecture.common import choices, nodes
+from tests.conjecture.common import choices, integer_constr, nodes
 
 
 @seed("deterministic so we don't miss some combination of features")
@@ -345,7 +345,7 @@ def test_fails(should_fail, should_fail_assume):
 def _decode_choice(value):
     if isinstance(value, list):
         if value[0] == "integer":
-            # large integers get cast to float, stored as ["integer", float(value)]
+            # large integers get cast to string, stored as ["integer", str(value)]
             assert isinstance(value[1], float)
             return int(value[1])
         elif value[0] == "bytes":
@@ -445,3 +445,47 @@ def test_nodes_json_roundtrips(nodes):
             assume(False)
     nodes2 = _decode_nodes(json.loads(json.dumps(nodes_to_json(nodes))))
     assert nodes == nodes2
+
+
+@pytest.mark.parametrize(
+    "choice, expected",
+    [
+        (math.nan, ["float", float_to_int(math.nan)]),
+        (SIGNALING_NAN, ["float", float_to_int(SIGNALING_NAN)]),
+        (1, 1),
+        (-1, -1),
+        (2**63 + 1, ["integer", str(2**63 + 1)]),
+        (-(2**63 + 1), ["integer", str(-(2**63 + 1))]),
+        (1.0, 1.0),
+        (-0.0, -0.0),
+        (0.0, 0.0),
+        (True, True),
+        (False, False),
+        (b"a", ["bytes", "YQ=="]),
+    ],
+)
+def test_choices_to_json_explicit(choice, expected):
+    assert choices_to_json([choice]) == [expected]
+
+
+@pytest.mark.parametrize(
+    "choice_node, expected",
+    [
+        (
+            ChoiceNode(
+                type="integer",
+                value=2**63 + 1,
+                constraints=integer_constr(),
+                was_forced=False,
+            ),
+            {
+                "type": "integer",
+                "value": ["integer", str(2**63 + 1)],
+                "constraints": integer_constr(),
+                "was_forced": False,
+            },
+        ),
+    ],
+)
+def test_choice_nodes_to_json_explicit(choice_node, expected):
+    assert nodes_to_json([choice_node]) == [expected]

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+RELEASE_TYPE: minor`
	`2`	`+`
	`3`	+This release adds \|OBSERVABILITY_CHOICE_NODES\|, a new option for :ref:`observability <observability>`, which includes the choice sequence in ``metadata.choice_nodes`` for test case observations if set.