Merge pull request #130 from huntergregory/fail-fast

add fail fast option for cli/generate

Author: mattfenwick

pkg/cli/generate.go

@@ -35,6 +35,7 @@ type GenerateArgs struct {
 	ServerNamespaces          []string
 	ServerPods                []string
 	CleanupNamespaces         bool
+	FailFast                  bool
 	Include                   []string
 	Exclude                   []string
 	DestinationType           string
@@ -72,6 +73,7 @@ func SetupGenerateCommand() *cobra.Command {
 	command.Flags().IntVar(&args.PodCreationTimeoutSeconds, "pod-creation-timeout-seconds", 60, "number of seconds to wait for pods to create, be running and have IP addresses")
 	command.Flags().StringVar(&args.Context, "context", "", "kubernetes context to use; if empty, uses default context")
 	command.Flags().BoolVar(&args.CleanupNamespaces, "cleanup-namespaces", false, "if true, clean up namespaces after completion")
+	command.Flags().BoolVar(&args.FailFast, "fail-fast", false, "if true, stop running tests after the first failure")
 	command.Flags().StringVar(&args.DestinationType, "destination-type", "", "override to set what to direct requests at; if not specified, the tests will be left as-is; one of "+strings.Join(generator.AllProbeModes, ", "))
 	command.Flags().IntVar(&args.JobTimeoutSeconds, "job-timeout-seconds", 10, "number of seconds to pass on to 'agnhost connect --timeout=%ds' flag")
 
@@ -121,6 +123,7 @@ func RunGenerateCommand(args *GenerateArgs) {
 		BatchJobs:                        batchJobs,
 		IgnoreLoopback:                   args.IgnoreLoopback,
 		JobTimeoutSeconds:                args.JobTimeoutSeconds,
+		FailFast:                         args.FailFast,
 	}
 	interpreter := connectivity.NewInterpreter(kubernetes, resources, interpreterConfig)
 	printer := &connectivity.Printer{
@@ -166,6 +169,11 @@ func RunGenerateCommand(args *GenerateArgs) {
 
 		printer.PrintTestCaseResult(result)
 		fmt.Printf("finished policy #%d\n", i+1)
+
+		if args.FailFast && !result.Passed(interpreter.Config.IgnoreLoopback) {
+			logrus.Warn("failing fast due to failure")
+			break
+		}
 	}
 
 	printer.PrintSummary()

pkg/connectivity/interpreter.go

@@ -2,14 +2,15 @@ package connectivity
 
 import (
 	"fmt"
+	"time"
+
 	"github.com/mattfenwick/cyclonus/pkg/connectivity/probe"
 	"github.com/mattfenwick/cyclonus/pkg/generator"
 	"github.com/mattfenwick/cyclonus/pkg/kube"
 	"github.com/mattfenwick/cyclonus/pkg/matcher"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	networkingv1 "k8s.io/api/networking/v1"
-	"time"
 )
 
 const (
@@ -27,6 +28,7 @@ type InterpreterConfig struct {
 	BatchJobs                        bool
 	IgnoreLoopback                   bool
 	JobTimeoutSeconds                int
+	FailFast                         bool
 }
 
 func (i *InterpreterConfig) PerturbationWaitDuration() time.Duration {
@@ -128,7 +130,12 @@ func (t *Interpreter) ExecuteTestCase(testCase *generator.TestCase) *Result {
 		logrus.Infof("step %d: waiting %d seconds for perturbation to take effect", stepIndex+1, t.Config.PerturbationWaitSeconds)
 		time.Sleep(t.Config.PerturbationWaitDuration())
 
-		result.Steps = append(result.Steps, t.runProbe(testCaseState, step.Probe))
+		stepResult := t.runProbe(testCaseState, step.Probe)
+		result.Steps = append(result.Steps, stepResult)
+
+		if t.Config.FailFast && !stepResult.Passed(t.Config.IgnoreLoopback) {
+			break
+		}
 	}
 
 	return result
@@ -151,7 +158,7 @@ func (t *Interpreter) runProbe(testCaseState *TestCaseState, probeConfig *genera
 		logrus.Infof("running kube probe on try %d", i+1)
 		stepResult.AddKubeProbe(t.kubeRunner.RunProbeForConfig(probeConfig, testCaseState.Resources))
 		// no differences between synthetic and kube probes?  then we can stop
-		if stepResult.LastComparison().ValueCounts(t.Config.IgnoreLoopback)[DifferentComparison] == 0 {
+		if stepResult.Passed(t.Config.IgnoreLoopback) {
 			break
 		}
 	}

pkg/connectivity/result.go

@@ -32,7 +32,7 @@ func (r *Result) Features() map[string][]string {
 
 func (r *Result) Passed(ignoreLoopback bool) bool {
 	for _, step := range r.Steps {
-		if step.LastComparison().ValueCounts(ignoreLoopback)[DifferentComparison] > 0 {
+		if !step.Passed(ignoreLoopback) {
 			return false
 		}
 	}

pkg/connectivity/stepresult.go

@@ -41,3 +41,7 @@ func (s *StepResult) LastComparison() *ComparisonTable {
 func (s *StepResult) LastKubeProbe() *probe.Table {
 	return s.KubeProbes[len(s.KubeProbes)-1]
 }
+
+func (s *StepResult) Passed(ignoreLoopback bool) bool {
+	return s.LastComparison().ValueCounts(ignoreLoopback)[DifferentComparison] == 0
+}