Skip to content

Commit b30adda

Browse files
tybugtybug
committed
document hypothesis observations metadata
1 parent 37e0960 commit b30adda

File tree

5 files changed

+110
-1
lines changed

5 files changed

+110
-1
lines changed

‎hypothesis-python/docs/prolog.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,17 +116,21 @@
116116
.. |PrimitiveProvider.draw_string| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_string`
117117
.. |PrimitiveProvider.draw_bytes| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_bytes`
118118
.. |PrimitiveProvider.on_observation| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.on_observation`
119+
.. |PrimitiveProvider.observe_test_case| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.observe_test_case`
120+
.. |PrimitiveProvider.observe_information_messages| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.observe_information_messages`
119121
.. |PrimitiveProvider.per_test_case_context_manager| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.per_test_case_context_manager`
120122
.. |PrimitiveProvider.add_observability_callback| replace:: :data:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.add_observability_callback`
121123

122124
.. |AVAILABLE_PROVIDERS| replace:: :data:`~hypothesis.internal.conjecture.providers.AVAILABLE_PROVIDERS`
123125
.. |TESTCASE_CALLBACKS| replace:: :data:`~hypothesis.internal.observability.TESTCASE_CALLBACKS`
126+
.. |OBSERVABILITY_CHOICE_NODES| replace:: :data:`~hypothesis.internal.observability.OBSERVABILITY_CHOICE_NODES`
124127
.. |BUFFER_SIZE| replace:: :data:`~hypothesis.internal.conjecture.engine.BUFFER_SIZE`
125128
.. |MAX_SHRINKS| replace:: :data:`~hypothesis.internal.conjecture.engine.MAX_SHRINKS`
126129
.. |MAX_SHRINKING_SECONDS| replace:: :data:`~hypothesis.internal.conjecture.engine.MAX_SHRINKING_SECONDS`
127130
.. |BackendCannotProceed| replace:: :exc:`~hypothesis.errors.BackendCannotProceed`
128131

129132
.. |@rule| replace:: :func:`@rule <hypothesis.stateful.rule>`
133+
.. |@precondition| replace:: :func:`@precondition <hypothesis.stateful.precondition>`
130134
.. |RuleBasedStateMachine| replace:: :class:`~hypothesis.stateful.RuleBasedStateMachine`
131135
.. |run_state_machine_as_test| replace:: :func:`~hypothesis.stateful.run_state_machine_as_test`
132136

‎hypothesis-python/docs/reference/integrations.rst

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,15 @@ including Gson in Java, ``JSON.parse()`` in Ruby, and of course in Python.
162162
:hide_key: /additionalProperties, /type
163163

164164

165+
Hypothesis Metadata
166+
^^^^^^^^^^^^^^^^^^^
167+
168+
While the observability format is agnostic to the property-based testing library which generated it, Hypothesis includes specific values in the ``metadata`` key for test cases. You may rely on these being present if and only if the observation was generated by Hypothesis.
169+
170+
.. jsonschema:: ./schema_metadata.json
171+
:hide_key: /additionalProperties, /type
172+
173+
165174
.. _pytest-plugin:
166175

167176
The Hypothesis pytest plugin

‎hypothesis-python/docs/reference/internals.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ Observability
3030

3131
.. autodata:: hypothesis.internal.observability.TESTCASE_CALLBACKS
3232
.. autodata:: hypothesis.internal.observability.OBSERVABILITY_COLLECT_COVERAGE
33-
33+
.. autodata:: hypothesis.internal.observability.OBSERVABILITY_CHOICE_NODES
3434

3535
Engine constants
3636
----------------

‎hypothesis-python/docs/reference/schema_metadata.json

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
{
2+
"title": "Hypothesis Metadata",
3+
"description": "Hypothesis-specific values included in the ``metadata`` key of observations for test cases.",
4+
"type": "object",
5+
"properties": {
6+
"traceback": {
7+
"type": ["string", "null"],
8+
"description": "The traceback for failing tests, if and only if ``status == \"failed\"``."
9+
},
10+
"reproduction_decorator": {
11+
"type": ["string", "null"],
12+
"description": "The ``@reproduce_failure`` decorator string for failing tests, if and only if ``status == \"failed\"``."
13+
},
14+
"predicates": {
15+
"type": "object",
16+
"description": "The number of times each |assume| and |@precondition| predicate was satisfied (``True``) and not satisfied (``False``).",
17+
"additionalProperties": {
18+
"type": "object",
19+
"properties": {
20+
"satisfied": {
21+
"type": "integer",
22+
"minimum": 0,
23+
"description": "The number of times this predicate was satisfied (``True``)."
24+
},
25+
"unsatisfied": {
26+
"type": "integer",
27+
"minimum": 0,
28+
"description": "The number of times this predicate was not satisfied (``False``)."
29+
}
30+
},
31+
"required": ["satisfied", "unsatisfied"],
32+
"additionalProperties": false
33+
}
34+
},
35+
"backend": {
36+
"type": "object",
37+
"description": "Backend-specific observations from |PrimitiveProvider.observe_test_case| and |PrimitiveProvider.observe_information_messages|."
38+
},
39+
"sys_argv": {
40+
"type": "array",
41+
"items": {"type": "string"},
42+
"description": "The result of ``sys.argv``."
43+
},
44+
"os_getpid": {
45+
"type": "integer",
46+
"description": "The result of ``os.getpid()``."
47+
},
48+
"imported_at": {
49+
"type": "number",
50+
"description": "The unix timestamp when Hypothesis was imported."
51+
},
52+
"data_status": {
53+
"type": "number",
54+
"enum": [0, 1, 2, 3],
55+
"description": "The internal status of the ConjectureData for this test case. The values are as follows: ``Status.OVERRUN = 0``, ``Status.INVALID = 1``, ``Status.VALID = 2``, and ``Status.INTERESTING = 3``."
56+
},
57+
"interesting_origin": {
58+
"type": ["string", "null"],
59+
"description": "The internal InterestingOrigin object for failing tests, if and only if ``status == \"failed\"``. The ``traceback`` string value is derived from this object."
60+
},
61+
"choice_nodes": {
62+
"type": ["array", "null"],
63+
"description": "The sequence of choices made during this test case. This includes the choice value, as well as its constraints and whether it was forced or not. The choice sequence is a relatively low-level implementation detail of Hypothesis, and is exposed here for users building tools or research on top of Hypothesis. See |PrimitiveProvider| for more details about the choice sequence.\n\nOnly present if |OBSERVABILITY_CHOICE_NODES| is set.",
64+
"items": {
65+
"type": "object",
66+
"properties": {
67+
"type": {
68+
"type": "string",
69+
"enum": ["integer", "float", "string", "bytes", "boolean"],
70+
"description": "The type of choice made. Corresponds to a call to |PrimitiveProvider.draw_integer|, |PrimitiveProvider.draw_float|, |PrimitiveProvider.draw_string|, |PrimitiveProvider.draw_bytes|, or |PrimitiveProvider.draw_boolean|."
71+
},
72+
"value": {
73+
"description": "The value of the choice. Corresponds to the value returned by a ``PrimitiveProvider.draw_*`` method. ``NaN`` values are returned as ``{\"float\": <float64_int_value>}``, to distinguish ``NaN``s with nonstandard bit patterns. Integers above ``2**53`` are returned as ``{\"integer\": float(value)}``, for compatibility with software with limitations on integer sizes."
74+
},
75+
"constraints": {
76+
"type": "object",
77+
"description": "The constraints for this choice. Corresponds to the constraints passed to a ``PrimitiveProvider.draw_*`` method."
78+
},
79+
"was_forced": {
80+
"type": "boolean",
81+
"description": "Whether this choice was forced. As an implementation detail, Hypothesis occasionally requires that some choices take on a specific value, for instance to end generation of collection elements early for performance. These values are \"forced\", and have ``was_forced = True``."
82+
}
83+
},
84+
"required": ["type", "value", "constraints", "was_forced"],
85+
"additionalProperties": false
86+
}
87+
}
88+
},
89+
"required": ["traceback", "reproduction_decorator", "predicates", "backend", "sys_argv", "os_getpid", "imported_at", "data_status", "interesting_origin", "choice_nodes"],
90+
"additionalProperties": false
91+
}

‎hypothesis-python/src/hypothesis/internal/observability.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,11 @@ def _system_metadata() -> dict[str, Any]:
352352
OBSERVABILITY_COLLECT_COVERAGE = (
353353
"HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_NOCOVER" not in os.environ
354354
)
355+
#: If ``True``, include the ``metadata.choice_nodes`` key in test case
356+
#: observations.
357+
#:
358+
#: ``False`` by default. ``metadata.choice_nodes`` can be substantial amount of
359+
#: data, and so must be opted-in to, even when observability is enabled.
355360
OBSERVABILITY_CHOICE_NODES = (
356361
"HYPOTHESIS_EXPERIMENTAL_OBSERVABILITY_CHOICE_NODES" in os.environ
357362
)

0 commit comments

Comments
 (0)