add choice_nodes, data_status, and interesting_origin to metadata

tybug · tybug · commit 2f636c97e0e4 · 2025-06-03T20:47:13.000-04:00
diff --git a/hypothesis-python/src/hypothesis/internal/observability.py b/hypothesis-python/src/hypothesis/internal/observability.py
@@ -10,7 +10,10 @@
 
 """Observability tools to spit out analysis-ready tables, one row per test case."""
 
+import base64
+import dataclasses
 import json
+import math
 import os
 import sys
 import time
@@ -20,10 +23,24 @@
 from dataclasses import dataclass
 from datetime import date, timedelta
 from functools import lru_cache
-from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Union, cast
 
 from hypothesis.configuration import storage_directory
 from hypothesis.errors import HypothesisWarning
+from hypothesis.internal.conjecture.choice import (
+    BooleanConstraints,
+    BytesConstraints,
+    ChoiceConstraintsT,
+    ChoiceNode,
+    ChoiceT,
+    ChoiceTypeT,
+    FloatConstraints,
+    IntegerConstraints,
+    StringConstraints,
+)
+from hypothesis.internal.escalation import InterestingOrigin
+from hypothesis.internal.floats import float_to_int
+from hypothesis.internal.intervalsets import IntervalSet
 
 if TYPE_CHECKING:
     from typing import TypeAlias
@@ -43,6 +60,89 @@ def update_count(self, *, condition: bool) -> None:
             self.unsatisfied += 1
 
 
+def _choice_to_json(choice: Union[ChoiceT, None]) -> Any:
+    if choice is None:
+        return None
+    # see the note on the same check in to_jsonable for why we cast large
+    # integers to floats.
+    if isinstance(choice, int) and not isinstance(choice, bool) and choice > 2**63:
+        return ["integer", float(choice)]
+    elif isinstance(choice, bytes):
+        return ["bytes", base64.b64encode(choice).decode()]
+    elif isinstance(choice, float) and math.isnan(choice):
+        # handle nonstandard nan bit patterns. We don't need to do this for -0.0
+        # vs 0.0 since json doesn't normalize -0.0 to 0.0.
+        return ["float", float_to_int(choice)]
+    return choice
+
+
+def choices_to_json(choices: tuple[ChoiceT, ...]) -> list[Any]:
+    return [_choice_to_json(choice) for choice in choices]
+
+
+def _constraints_to_json(
+    choice_type: ChoiceTypeT, constraints: ChoiceConstraintsT
+) -> dict[str, Any]:
+    constraints = constraints.copy()
+    if choice_type == "integer":
+        constraints = cast(IntegerConstraints, constraints)
+        return {
+            "min_value": _choice_to_json(constraints["min_value"]),
+            "max_value": _choice_to_json(constraints["max_value"]),
+            "weights": (
+                None
+                if constraints["weights"] is None
+                # wrap up in a list, instead of a dict, because json dicts
+                # require string keys
+                else [
+                    (_choice_to_json(k), v) for k, v in constraints["weights"].items()
+                ]
+            ),
+            "shrink_towards": _choice_to_json(constraints["shrink_towards"]),
+        }
+    elif choice_type == "float":
+        constraints = cast(FloatConstraints, constraints)
+        return {
+            "min_value": _choice_to_json(constraints["min_value"]),
+            "max_value": _choice_to_json(constraints["max_value"]),
+            "allow_nan": constraints["allow_nan"],
+            "smallest_nonzero_magnitude": constraints["smallest_nonzero_magnitude"],
+        }
+    elif choice_type == "string":
+        constraints = cast(StringConstraints, constraints)
+        assert isinstance(constraints["intervals"], IntervalSet)
+        return {
+            "intervals": constraints["intervals"].intervals,
+            "min_size": _choice_to_json(constraints["min_size"]),
+            "max_size": _choice_to_json(constraints["max_size"]),
+        }
+    elif choice_type == "bytes":
+        constraints = cast(BytesConstraints, constraints)
+        return {
+            "min_size": _choice_to_json(constraints["min_size"]),
+            "max_size": _choice_to_json(constraints["max_size"]),
+        }
+    elif choice_type == "boolean":
+        constraints = cast(BooleanConstraints, constraints)
+        return {
+            "p": constraints["p"],
+        }
+    else:
+        raise NotImplementedError(f"unknown choice type {choice_type}")
+
+
+def nodes_to_json(nodes: tuple[ChoiceNode, ...]) -> list[dict[str, Any]]:
+    return [
+        {
+            "type": node.type,
+            "value": _choice_to_json(node.value),
+            "constraints": _constraints_to_json(node.type, node.constraints),
+            "was_forced": node.was_forced,
+        }
+        for node in nodes
+    ]
+
+
 @dataclass
 class ObservationMetadata:
     traceback: Optional[str]
@@ -52,6 +152,28 @@ class ObservationMetadata:
     sys_argv: list[str]
     os_getpid: int
     imported_at: float
+    data_status: "Status"
+    interesting_origin: Optional[InterestingOrigin]
+    choice_nodes: Optional[tuple[ChoiceNode, ...]]
+
+    def to_json(self) -> dict[str, Any]:
+        data = {
+            "traceback": self.traceback,
+            "reproduction_decorator": self.reproduction_decorator,
+            "predicates": self.predicates,
+            "backend": self.backend,
+            "sys.argv": self.sys_argv,
+            "os.getpid()": self.os_getpid,
+            "imported_at": self.imported_at,
+            "data_status": self.data_status,
+            "interesting_origin": self.interesting_origin,
+            "choice_nodes": (
+                None if self.choice_nodes is None else nodes_to_json(self.choice_nodes)
+            ),
+        }
+        # check that we didn't forget one
+        assert len(data) == len(dataclasses.fields(self))
+        return data
 
 
 @dataclass
@@ -183,6 +305,9 @@ def make_testcase(
                 ),
                 "predicates": dict(data._observability_predicates),
                 "backend": backend_metadata or {},
+                "data_status": data.status,
+                "interesting_origin": data.interesting_origin,
+                "choice_nodes": data.nodes if OBSERVABILITY_CHOICE_NODES else None,
                 **_system_metadata(),
                 # unpack last so it takes precedence for duplicate keys
                 **(metadata or {}),
@@ -204,11 +329,7 @@ def _deliver_to_file(observation: Observation) -> None:  # pragma: no cover
     fname.parent.mkdir(exist_ok=True, parents=True)
     _WROTE_TO.add(fname)
     with fname.open(mode="a") as f:
-        obs_json: dict[str, Any] = to_jsonable(observation, avoid_realization=False)  # type: ignore
-        if obs_json["type"] == "test_case":
-            obs_json["metadata"]["sys.argv"] = obs_json["metadata"].pop("sys_argv")
-            obs_json["metadata"]["os.getpid()"] = obs_json["metadata"].pop("os_getpid")
-        f.write(json.dumps(obs_json) + "\n")
+        f.write(json.dumps(to_jsonable(observation, avoid_realization=False)) + "\n")
 
 
 _imported_at = time.time()
@@ -231,6 +352,10 @@ def _system_metadata() -> dict[str, Any]:
 OBSERVABILITY_COLLECT_COVERAGE = (
     "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_NOCOVER" not in os.environ
 )
+OBSERVABILITY_CHOICE_NODES = (
+    "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_CHOICE_NODES" in os.environ
+)
+
 if OBSERVABILITY_COLLECT_COVERAGE is False and (
     sys.version_info[:2] >= (3, 12)
 ):  # pragma: no cover
@@ -240,8 +365,10 @@ def _system_metadata() -> dict[str, Any]:
         HypothesisWarning,
         stacklevel=2,
     )
-if "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY" in os.environ or (
-    OBSERVABILITY_COLLECT_COVERAGE is False
+
+if (
+    "HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY" in os.environ
+    or OBSERVABILITY_COLLECT_COVERAGE is False
 ):  # pragma: no cover
     TESTCASE_CALLBACKS.append(_deliver_to_file)
 
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/utils.py b/hypothesis-python/src/hypothesis/strategies/_internal/utils.py
@@ -165,6 +165,10 @@ def to_jsonable(obj: object, *, avoid_realization: bool) -> object:
     known types.
     """
     if isinstance(obj, (str, int, float, bool, type(None))):
+        # We convert integers of 2**63 to floats, to avoid crashing external
+        # utilities with a 64 bit integer cap (notable, sqlite). See
+        # https://github.com/HypothesisWorks/hypothesis/pull/3797#discussion_r1413425110
+        # and https://github.com/simonw/sqlite-utils/issues/605.
         if isinstance(obj, int) and not isinstance(obj, bool) and abs(obj) >= 2**63:
             # Silently clamp very large ints to max_float, to avoid OverflowError when
             # casting to float.  (but avoid adding more constraints to symbolic values)
diff --git a/hypothesis-python/tests/conjecture/common.py b/hypothesis-python/tests/conjecture/common.py
@@ -394,6 +394,12 @@ def draw_value(choice_type, constraints):
     return getattr(data, f"draw_{choice_type}")(**constraints)
 
 
+@st.composite
+def choices(draw):
+    (choice_type, constraints) = draw(choice_types_constraints())
+    return draw_value(choice_type, constraints)
+
+
 @st.composite
 def nodes(draw, *, was_forced=None, choice_types=None):
     if choice_types is None:
diff --git a/hypothesis-python/tests/cover/test_observability.py b/hypothesis-python/tests/cover/test_observability.py
@@ -8,6 +8,9 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import base64
+import json
+import math
 import textwrap
 from contextlib import nullcontext
 
@@ -26,7 +29,11 @@
 )
 from hypothesis.database import InMemoryExampleDatabase
 from hypothesis.internal.compat import PYPY
+from hypothesis.internal.conjecture.choice import ChoiceNode, choices_key
 from hypothesis.internal.coverage import IN_COVERAGE_TESTS
+from hypothesis.internal.floats import SIGNALING_NAN, int_to_float
+from hypothesis.internal.intervalsets import IntervalSet
+from hypothesis.internal.observability import choices_to_json, nodes_to_json
 from hypothesis.stateful import (
     RuleBasedStateMachine,
     invariant,
@@ -35,6 +42,7 @@
 )
 
 from tests.common.utils import Why, capture_observations, xfail_on_crosshair
+from tests.conjecture.common import choices, nodes
 
 
 @seed("deterministic so we don't miss some combination of features")
@@ -332,3 +340,108 @@ def test_fails(should_fail, should_fail_assume):
     assert len(ls) == 1
     assert ls[0].status == expected_status
     assert ls[0].how_generated == "fuzz_one_input"
+
+
+def _decode_choice(value):
+    if isinstance(value, list):
+        if value[0] == "integer":
+            # large integers get cast to float, stored as ["integer", float(value)]
+            assert isinstance(value[1], float)
+            return int(value[1])
+        elif value[0] == "bytes":
+            assert isinstance(value[1], str)
+            return base64.b64decode(value[1])
+        elif value[0] == "float":
+            assert isinstance(value[1], int)
+            choice = int_to_float(value[1])
+            assert math.isnan(choice)
+            return choice
+        else:
+            return value[1]
+
+    return value
+
+
+def _decode_choices(data):
+    return [_decode_choice(value) for value in data]
+
+
+def _decode_nodes(data):
+    return [
+        ChoiceNode(
+            type=node["type"],
+            value=_decode_choice(node["value"]),
+            constraints=_decode_constraints(node["type"], node["constraints"]),
+            was_forced=node["was_forced"],
+        )
+        for node in data
+    ]
+
+
+def _decode_constraints(choice_type, data):
+    if choice_type == "integer":
+        return {
+            "min_value": _decode_choice(data["min_value"]),
+            "max_value": _decode_choice(data["max_value"]),
+            "weights": (
+                None
+                if data["weights"] is None
+                else {_decode_choice(k): v for k, v in data["weights"]}
+            ),
+            "shrink_towards": _decode_choice(data["shrink_towards"]),
+        }
+    elif choice_type == "float":
+        return {
+            "min_value": _decode_choice(data["min_value"]),
+            "max_value": _decode_choice(data["max_value"]),
+            "allow_nan": data["allow_nan"],
+            "smallest_nonzero_magnitude": data["smallest_nonzero_magnitude"],
+        }
+    elif choice_type == "string":
+        return {
+            "intervals": IntervalSet(tuple(data["intervals"])),
+            "min_size": _decode_choice(data["min_size"]),
+            "max_size": _decode_choice(data["max_size"]),
+        }
+    elif choice_type == "bytes":
+        return {
+            "min_size": _decode_choice(data["min_size"]),
+            "max_size": _decode_choice(data["max_size"]),
+        }
+    elif choice_type == "boolean":
+        return {"p": data["p"]}
+    else:
+        raise ValueError(f"unknown choice type {choice_type}")
+
+
+def _will_be_cast_to_float(value):
+    return isinstance(value, int) and abs(value) >= 2**63
+
+
+@example([0.0])
+@example([-0.0])
+@example([SIGNALING_NAN])
+@example([math.nan])
+@example([math.inf])
+@example([-math.inf])
+@given(st.lists(choices()))
+def test_choices_json_roundtrips(choices):
+    # choices_to_json and nodes_to_json roundtrip, *except for large integers*,
+    # which get cast to the nearest integer-valued float on roundtrip. This is
+    # an intentional design decision of the format; see related comment in
+    # to_jsonable.
+    if any(_will_be_cast_to_float(choice) for choice in choices):
+        assume(False)
+    choices2 = _decode_choices(json.loads(json.dumps(choices_to_json(choices))))
+    assert choices_key(choices) == choices_key(choices2)
+
+
+@given(st.lists(nodes()))
+def test_nodes_json_roundtrips(nodes):
+    for node in nodes:
+        if _will_be_cast_to_float(node.value) or any(
+            _will_be_cast_to_float(value) for value in node.constraints.values()
+        ):
+            assume(False)
+    nodes2 = _decode_nodes(json.loads(json.dumps(nodes_to_json(nodes))))
+    assert nodes == nodes2
