Implement swarm testing and use it for rule based stateful tests

hypothesis-python/RELEASE.rst

@@ -0,0 +1,8 @@
+RELEASE_TYPE: minor
+
+This release significantly improves the data distribution in rule based stateful testing <stateful_testing>,
+by using a technique called `Swarm Testing (Groce, Alex, et al. "Swarm testing."
+Proceedings of the 2012 International Symposium on Software Testing and Analysis. ACM, 2012.) <https://agroce.github.io/issta12.pdf>`_
+to select which rules are run in any given test case. This should allow it to find many issues that it would previously have missed.
+
+This change is likely to be especially beneficial for stateful tests with large numbers of rules.

hypothesis-python/src/hypothesis/core.py

@@ -758,6 +758,12 @@ def run_engine(self):
                 report("".join(traceback.format_exception(type(e), e, tb)))
 
             finally:  # pragma: no cover
+                # Mostly useful for ``find`` and ensuring that objects that
+                # hold on to a reference to ``data`` know that it's now been
+                # finished and they shouldn't attempt to draw more data from
+                # it.
+                ran_example.freeze()
+
                 # This section is in fact entirely covered by the tests in
                 # test_reproduce_failure, but it seems to trigger a lovely set
                 # of coverage bugs: The branches show up as uncovered (despite

hypothesis-python/src/hypothesis/searchstrategy/featureflags.py

@@ -0,0 +1,129 @@
+# coding=utf-8
+#
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2019 David R. MacIver
+# ([email protected]), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+from __future__ import absolute_import, division, print_function
+
+import hypothesis.internal.conjecture.utils as cu
+from hypothesis.searchstrategy.strategies import SearchStrategy
+
+FEATURE_LABEL = cu.calc_label_from_name("feature flag")
+
+
+class FeatureFlags(object):
+    """Object that can be used to control a number of feature flags for a
+    given test run.
+
+    This enables an approach to data generation called swarm testing (
+    see Groce, Alex, et al. "Swarm testing." Proceedings of the 2012
+    International Symposium on Software Testing and Analysis. ACM, 2012), in
+    which generation is biased by selectively turning some features off for
+    each test case generated. When there are many interacting features this can
+    find bugs that a pure generation strategy would otherwise have missed.
+
+    FeatureFlags are designed to "shrink open", so that during shrinking they
+    become less restrictive. This allows us to potentially shrink to smaller
+    test cases that were forbidden during the generation phase because they
+    required disabled features.
+    """
+
+    def __init__(self, data=None, enabled=(), disabled=()):
+        self.__data = data
+        self.__decisions = {}
+
+        for f in enabled:
+            self.__decisions[f] = 0
+
+        for f in disabled:
+            self.__decisions[f] = 255
+
+        # In the original swarm testing paper they turn features on or off
+        # uniformly at random. Instead we decide the probability with which to
+        # enable features up front. This can allow for scenarios where all or
+        # no features are enabled, which are vanishingly unlikely in the
+        # original model.
+        #
+        # We implement this as a single 8-bit integer and enable features which
+        # score >= that value. In particular when self.__baseline is 0, all
+        # features will be enabled. This is so that we shrink in the direction
+        # of more features being enabled.
+        if self.__data is not None:
+            self.__baseline = data.draw_bits(8)
+        else:
+            # If data is None we're in example mode so all that matters is the
+            # enabled/disabled lists above. We set this up so that
+            self.__baseline = 1
+
+    def is_enabled(self, name):
+        """Tests whether the feature named ``name`` should be enabled on this
+        test run."""
+        if self.__data is None or self.__data.frozen:
+            # Feature set objects might hang around after data generation has
+            # finished. If this happens then we just report all new features as
+            # enabled, because that's our shrinking direction and they have no
+            # impact on data generation if they weren't used while it was
+            # running.
+            try:
+                return self.__is_value_enabled(self.__decisions[name])
+            except KeyError:
+                return True
+
+        data = self.__data
+
+        data.start_example(label=FEATURE_LABEL)
+        if name in self.__decisions:
+            # If we've already decided on this feature then we don't actually
+            # need to draw anything, but we do write the same decision to the
+            # input stream. This allows us to lazily decide whether a feature
+            # is enabled, because it means that if we happen to delete the part
+            # of the test case where we originally decided, the next point at
+            # which we make this decision just makes the decision it previously
+            # made.
+            value = self.__decisions[name]
+            data.draw_bits(8, forced=value)
+        else:
+            # If the baseline is 0 then everything is enabled so it doesn't
+            # matter what we have here and we might as well make the shrinker's
+            # life easier by forcing it to zero.
+            if self.__baseline == 0:
+                value = 0
+                data.draw_bits(8, forced=0)
+            else:
+                value = data.draw_bits(8)
+            self.__decisions[name] = value
+        data.stop_example()
+        return self.__is_value_enabled(value)
+
+    def __is_value_enabled(self, value):
+        """Check if a given value drawn for a feature counts as enabled. Note
+        that low values are more likely to be enabled. This is again in aid of
+        shrinking open. In particular a value of 255 is always enabled."""
+        return (255 - value) >= self.__baseline
+
+    def __repr__(self):
+        enabled = []
+        disabled = []
+        for k, v in self.__decisions.items():
+            if self.__is_value_enabled(v):
+                enabled.append(k)
+            else:
+                disabled.append(k)
+        return "FeatureFlags(enabled=%r, disabled=%r)" % (enabled, disabled)
+
+
+class FeatureStrategy(SearchStrategy):
+    def do_draw(self, data):
+        return FeatureFlags(data)

hypothesis-python/src/hypothesis/stateful.py

@@ -47,6 +47,7 @@
 from hypothesis.internal.reflection import function_digest, nicerepr, proxies, qualname
 from hypothesis.internal.validation import check_type
 from hypothesis.reporting import current_verbosity, report
+from hypothesis.searchstrategy.featureflags import FeatureStrategy
 from hypothesis.searchstrategy.strategies import OneOfStrategy, SearchStrategy
 from hypothesis.vendor.pretty import CUnicodeIO, RepresentationPrinter
 
@@ -610,6 +611,10 @@ def __init__(self, machine):
         self.machine = machine
         self.rules = list(machine.rules())
 
+        self.enabled_rules_strategy = st.shared(
+            FeatureStrategy(), key=("enabled rules", machine),
+        )
+
         # The order is a bit arbitrary. Primarily we're trying to group rules
         # that write to the same location together, and to put rules with no
         # target first as they have less effect on the structure. We order from
@@ -635,12 +640,27 @@ def do_draw(self, data):
         if not any(self.is_valid(rule) for rule in self.rules):
             msg = u"No progress can be made from state %r" % (self.machine,)
             quiet_raise(InvalidDefinition(msg))
-        rule = data.draw(st.sampled_from(self.rules).filter(self.is_valid))
+
+        feature_flags = data.draw(self.enabled_rules_strategy)
+
+        # Note: The order of the filters here is actually quite important,
+        # because checking is_enabled makes choices, so increases the size of
+        # the choice sequence. This means that if we are in a case where many
+        # rules are invalid we will make a lot more choices if we ask if they
+        # are enabled before we ask if they are valid, so our test cases will
+        # be artificially large.
+        rule = data.draw(
+            st.sampled_from(self.rules)
+            .filter(self.is_valid)
+            .filter(lambda r: feature_flags.is_enabled(r.function.__name__))
+        )
+
         return (rule, data.draw(rule.arguments_strategy))
 
     def is_valid(self, rule):
         if rule.precondition and not rule.precondition(self.machine):
             return False
+
         for b in rule.bundles:
             bundle = self.machine.bundle(b.name)
             if not bundle:

hypothesis-python/tests/cover/test_feature_flags.py

@@ -0,0 +1,84 @@
+# coding=utf-8
+#
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Most of this work is copyright (C) 2013-2019 David R. MacIver
+# ([email protected]), but it contains contributions by others. See
+# CONTRIBUTING.rst for a full list of people who may hold copyright, and
+# consult the git log if you need to determine who owns an individual
+# contribution.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+#
+# END HEADER
+
+from __future__ import absolute_import, division, print_function
+
+from hypothesis import given, strategies as st
+from hypothesis.internal.compat import hrange
+from hypothesis.searchstrategy.featureflags import FeatureFlags, FeatureStrategy
+from tests.common.debug import find_any, minimal
+
+STRAT = FeatureStrategy()
+
+
+def test_can_all_be_enabled():
+    find_any(STRAT, lambda x: all(x.is_enabled(i) for i in hrange(100)))
+
+
+def test_can_all_be_disabled():
+    find_any(STRAT, lambda x: all(not x.is_enabled(i) for i in hrange(100)))
+
+
+def test_minimizes_open():
+    features = hrange(10)
+
+    flags = minimal(STRAT, lambda x: [x.is_enabled(i) for i in features])
+
+    assert all(flags.is_enabled(i) for i in features)
+
+
+def test_minimizes_individual_features_to_open():
+    features = list(hrange(10))
+
+    flags = minimal(
+        STRAT, lambda x: sum([x.is_enabled(i) for i in features]) < len(features)
+    )
+
+    assert all(flags.is_enabled(i) for i in features[:-1])
+    assert not flags.is_enabled(features[-1])
+
+
+def test_marks_unknown_features_as_enabled():
+    x = find_any(STRAT, lambda v: True)
+
+    assert x.is_enabled("fish")
+
+
+def test_by_default_all_enabled():
+    f = FeatureFlags()
+
+    assert f.is_enabled("foo")
+
+
+@given(st.data())
+def test_repr_can_be_evalled(data):
+    flags = data.draw(STRAT)
+
+    features = data.draw(st.lists(st.text(), unique=True))
+
+    for f in features:
+        flags.is_enabled(f)
+
+    flags2 = eval(repr(flags))
+
+    for f in features:
+        assert flags2.is_enabled(f) == flags.is_enabled(f)
+
+    more_features = data.draw(st.lists(st.text().filter(lambda s: s not in features)))
+
+    for f in more_features:
+        assert flags2.is_enabled(f)

hypothesis-python/tests/cover/test_stateful.py

@@ -223,12 +223,42 @@ def fail(self, x, y):
         assert False
 
 
+class CanSwarm(RuleBasedStateMachine):
+    """This test will essentially never pass if you choose rules uniformly at
+    random, because every time the snake rule fires we return to the beginning,
+    so we will tend to undo progress well before we make enough progress for
+    the test to fail.
+
+    This tests our swarm testing functionality in stateful testing by ensuring
+    that we can sometimes generate long runs of steps which exclude a
+    particular rule.
+    """
+
+    def __init__(self):
+        super(CanSwarm, self).__init__()
+        self.seen = set()
+
+    # The reason this rule takes a parameter is that it ensures that we do not
+    # achieve "swarming" by by just restricting the alphabet for single byte
+    # decisions, which is a thing the underlying conjecture engine  will
+    # happily do on its own without knowledge of the rule structure.
+    @rule(move=integers(0, 255))
+    def ladder(self, move):
+        self.seen.add(move)
+        assert len(self.seen) <= 15
+
+    @rule()
+    def snake(self):
+        self.seen.clear()
+
+
 bad_machines = (
     BalancedTrees,
     DepthMachine,
     RoseTreeStateMachine,
     NotTheLastMachine,
     PopulateMultipleTargets,
+    CanSwarm,
 )
 
 for m in bad_machines:
@@ -1164,7 +1194,7 @@ def oops(self):
 
 
 def test_reproduce_failure_works():
-    @reproduce_failure(__version__, base64.b64encode(b"\0\0\0"))
+    @reproduce_failure(__version__, base64.b64encode(b"\0\0\0\0\0"))
     class TrivialMachine(RuleBasedStateMachine):
         @rule()
         def oops(self):
@@ -1175,7 +1205,7 @@ def oops(self):
 
 
 def test_reproduce_failure_fails_if_no_error():
-    @reproduce_failure(__version__, base64.b64encode(b"\0\0\0"))
+    @reproduce_failure(__version__, base64.b64encode(b"\0\0\0\0\0"))
     class TrivialMachine(RuleBasedStateMachine):
         @rule()
         def ok(self):

hypothesis-python/tests/cover/test_statistical_events.py

@@ -234,4 +234,4 @@ def do(self, item):
 
 def test_stateful_states_are_deduped():
     stats = call_for_statistics(DemoStateMachine.TestCase().runTest)
-    assert len(stats.events) == 1
+    assert len(stats.events) <= 2