feat: add failfast support per test (#1745) (#1746)

Signed-off-by: Charles-Edouard Brétéché <[email protected]>
Co-authored-by: Charles-Edouard Brétéché <[email protected]>

Author: gcp-cherry-pick-bot[bot]

.crds/chainsaw.kyverno.io_tests.yaml

@@ -810,6 +810,10 @@ spec:
               description:
                 description: Description contains a description of the test.
                 type: string
+              failFast:
+                description: FailFast determines whether the test should stop upon
+                  encountering the first failure.
+                type: boolean
               forceTerminationGracePeriod:
                 description: ForceTerminationGracePeriod forces the termination grace
                   period on pods, statefulsets, daemonsets and deployments.

.release-notes/main.md

@@ -23,6 +23,7 @@ Release notes for `TODO`.
     - `null` means inherit the current cluster
     - an empty string means the default cluster
 - Made default timeouts part of the schemas
+- Added support to fail fast at the test level (test will be skipped if a previous error was already reported)
 
 ## 🔧 Fixes 🔧
 

.schemas/json/test-chainsaw-v1alpha1.json

@@ -1592,6 +1592,13 @@
             "null"
           ]
         },
+        "failFast": {
+          "description": "FailFast determines whether the test should stop upon encountering the first failure.",
+          "type": [
+            "boolean",
+            "null"
+          ]
+        },
         "forceTerminationGracePeriod": {
           "description": "ForceTerminationGracePeriod forces the termination grace period on pods, statefulsets, daemonsets and deployments.",
           "type": [

pkg/apis/v1alpha1/test.go

@@ -28,6 +28,10 @@ type TestSpec struct {
 	// +optional
 	Description string `json:"description,omitempty"`
 
+	// FailFast determines whether the test should stop upon encountering the first failure.
+	// +optional
+	FailFast *bool `json:"failFast,omitempty"`
+
 	// Timeouts for the test. Overrides the global timeouts set in the Configuration on a per operation basis.
 	// +optional
 	Timeouts *Timeouts `json:"timeouts,omitempty"`

pkg/apis/v1alpha1/zz_generated.deepcopy.go

@@ -810,6 +810,10 @@ spec:
               description:
                 description: Description contains a description of the test.
                 type: string
+              failFast:
+                description: FailFast determines whether the test should stop upon
+                  encountering the first failure.
+                type: boolean
               forceTerminationGracePeriod:
                 description: ForceTerminationGracePeriod forces the termination grace
                   period on pods, statefulsets, daemonsets and deployments.

pkg/data/crds/chainsaw.kyverno.io_tests.yaml

@@ -1592,6 +1592,13 @@
             "null"
           ]
         },
+        "failFast": {
+          "description": "FailFast determines whether the test should stop upon encountering the first failure.",
+          "type": [
+            "boolean",
+            "null"
+          ]
+        },
         "forceTerminationGracePeriod": {
           "description": "ForceTerminationGracePeriod forces the termination grace period on pods, statefulsets, daemonsets and deployments.",
           "type": [

pkg/data/schemas/json/test-chainsaw-v1alpha1.json

@@ -88,7 +88,11 @@ func (p *testsProcessor) Run(ctx context.Context, tc model.TestContext, tests ..
 				if test.Test.Spec.Skip != nil && *test.Test.Spec.Skip {
 					t.SkipNow()
 				}
-				if p.config.Execution.FailFast {
+				failFast := p.config.Execution.FailFast
+				if test.Test.Spec.FailFast != nil {
+					failFast = *test.Test.Spec.FailFast
+				}
+				if failFast {
 					if tc.Failed() > 0 {
 						t.SkipNow()
 					}

pkg/runner/processors/tests.go

@@ -852,6 +852,7 @@ If a resource doesn't exist yet in the cluster it will fail.</p>
 | Field | Type | Required | Inline | Description |
 |---|---|---|---|---|
 | `description` | `string` |  |  | <p>Description contains a description of the test.</p> |
+| `failFast` | `bool` |  |  | <p>FailFast determines whether the test should stop upon encountering the first failure.</p> |
 | `timeouts` | [`Timeouts`](#chainsaw-kyverno-io-v1alpha1-Timeouts) |  |  | <p>Timeouts for the test. Overrides the global timeouts set in the Configuration on a per operation basis.</p> |
 | `cluster` | `string` |  |  | <p>Cluster defines the target cluster (default cluster will be used if not specified and/or overridden).</p> |
 | `clusters` | [`Clusters`](#chainsaw-kyverno-io-v1alpha1-Clusters) |  |  | <p>Clusters holds a registry to clusters to support multi-cluster tests.</p> |