refactor: junit report structure (cherry-pick #1977)

.crds/chainsaw.kyverno.io_configurations.yaml

@@ -831,11 +831,14 @@ spec:
                 type: integer
               reportFormat:
                 description: |-
-                  ReportFormat determines test report format (JSON|XML|nil) nil == no report.
+                  ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION|nil) nil == no report.
                   maps to report.Type, however we don't want generated.deepcopy to have reference to it.
                 enum:
                 - JSON
                 - XML
+                - JUNIT-TEST
+                - JUNIT-STEP
+                - JUNIT-OPERATION
                 type: string
               reportName:
                 default: chainsaw-report
@@ -1761,10 +1764,13 @@ spec:
                 properties:
                   format:
                     default: JSON
-                    description: ReportFormat determines test report format (JSON|XML).
+                    description: ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION).
                     enum:
                     - JSON
                     - XML
+                    - JUNIT-TEST
+                    - JUNIT-STEP
+                    - JUNIT-OPERATION
                     type: string
                   name:
                     default: chainsaw-report

.schemas/json/configuration-chainsaw-v1alpha1.json

@@ -1659,14 +1659,17 @@
           "minimum": 1
         },
         "reportFormat": {
-          "description": "ReportFormat determines test report format (JSON|XML|nil) nil == no report.\nmaps to report.Type, however we don't want generated.deepcopy to have reference to it.",
+          "description": "ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION|nil) nil == no report.\nmaps to report.Type, however we don't want generated.deepcopy to have reference to it.",
           "type": [
             "string",
             "null"
           ],
           "enum": [
             "JSON",
-            "XML"
+            "XML",
+            "JUNIT-TEST",
+            "JUNIT-STEP",
+            "JUNIT-OPERATION"
           ]
         },
         "reportName": {

.schemas/json/configuration-chainsaw-v1alpha2.json

@@ -1747,15 +1747,18 @@
           ],
           "properties": {
             "format": {
-              "description": "ReportFormat determines test report format (JSON|XML).",
+              "description": "ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION).",
               "type": [
                 "string",
                 "null"
               ],
               "default": "JSON",
               "enum": [
                 "JSON",
-                "XML"
+                "XML",
+                "JUNIT-TEST",
+                "JUNIT-STEP",
+                "JUNIT-OPERATION"
               ]
             },
             "name": {

pkg/apis/v1alpha1/configuration.go

@@ -54,10 +54,10 @@ type ConfigurationSpec struct {
 	// +kubebuilder:default:=Background
 	DeletionPropagationPolicy metav1.DeletionPropagation `json:"deletionPropagationPolicy,omitempty"`
 
-	// ReportFormat determines test report format (JSON|XML|nil) nil == no report.
+	// ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION|nil) nil == no report.
 	// maps to report.Type, however we don't want generated.deepcopy to have reference to it.
 	// +optional
-	// +kubebuilder:validation:Enum:=JSON;XML;
+	// +kubebuilder:validation:Enum:=JSON;XML;JUNIT-TEST;JUNIT-STEP;JUNIT-OPERATION;
 	ReportFormat ReportFormatType `json:"reportFormat,omitempty"`
 
 	// ReportPath defines the path.
@@ -124,7 +124,10 @@ type ConfigurationSpec struct {
 type ReportFormatType string
 
 const (
-	JSONFormat ReportFormatType = "JSON"
-	XMLFormat  ReportFormatType = "XML"
-	NoReport   ReportFormatType = ""
+	JSONFormat           ReportFormatType = "JSON"
+	XMLFormat            ReportFormatType = "XML"
+	JUnitTestFormat      ReportFormatType = "JUNIT-TEST"
+	JUnitStepFormat      ReportFormatType = "JUNIT-STEP"
+	JUnitOperationFormat ReportFormatType = "JUNIT-OPERATION"
+	NoReport             ReportFormatType = ""
 )

pkg/apis/v1alpha2/options.go

@@ -95,15 +95,18 @@ type NamespaceOptions struct {
 type ReportFormatType string
 
 const (
-	JSONFormat ReportFormatType = "JSON"
-	XMLFormat  ReportFormatType = "XML"
+	JSONFormat           ReportFormatType = "JSON"
+	XMLFormat            ReportFormatType = "XML"
+	JUnitTestFormat      ReportFormatType = "JUNIT-TEST"
+	JUnitStepFormat      ReportFormatType = "JUNIT-STEP"
+	JUnitOperationFormat ReportFormatType = "JUNIT-OPERATION"
 )
 
 // ReportOptions contains the configuration used for reporting.
 type ReportOptions struct {
-	// ReportFormat determines test report format (JSON|XML).
+	// ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION).
 	// +optional
-	// +kubebuilder:validation:Enum:=JSON;XML
+	// +kubebuilder:validation:Enum:=JSON;XML;JUNIT-TEST;JUNIT-STEP;JUNIT-OPERATION
 	// +kubebuilder:default:="JSON"
 	Format ReportFormatType `json:"format,omitempty"`
 

pkg/commands/test/command.go

@@ -365,7 +365,7 @@ func Command() *cobra.Command {
 	cmd.Flags().StringVar(&options.deletionPropagationPolicy, "deletion-propagation-policy", "Background", "The deletion propagation policy (Foreground|Background|Orphan)")
 	// error options
 	// reporting options
-	cmd.Flags().StringVar(&options.reportFormat, "report-format", "", "Test report format (JSON|XML|nil)")
+	cmd.Flags().StringVar(&options.reportFormat, "report-format", "", "Test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION)")
 	cmd.Flags().StringVar(&options.reportName, "report-name", "chainsaw-report", "The name of the report to create")
 	cmd.Flags().StringVar(&options.reportPath, "report-path", "", "The path of the report to create")
 	// multi-cluster options

pkg/data/crds/chainsaw.kyverno.io_configurations.yaml

@@ -831,11 +831,14 @@ spec:
                 type: integer
               reportFormat:
                 description: |-
-                  ReportFormat determines test report format (JSON|XML|nil) nil == no report.
+                  ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION|nil) nil == no report.
                   maps to report.Type, however we don't want generated.deepcopy to have reference to it.
                 enum:
                 - JSON
                 - XML
+                - JUNIT-TEST
+                - JUNIT-STEP
+                - JUNIT-OPERATION
                 type: string
               reportName:
                 default: chainsaw-report
@@ -1761,10 +1764,13 @@ spec:
                 properties:
                   format:
                     default: JSON
-                    description: ReportFormat determines test report format (JSON|XML).
+                    description: ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION).
                     enum:
                     - JSON
                     - XML
+                    - JUNIT-TEST
+                    - JUNIT-STEP
+                    - JUNIT-OPERATION
                     type: string
                   name:
                     default: chainsaw-report

pkg/data/schemas/json/configuration-chainsaw-v1alpha1.json

@@ -1659,14 +1659,17 @@
           "minimum": 1
         },
         "reportFormat": {
-          "description": "ReportFormat determines test report format (JSON|XML|nil) nil == no report.\nmaps to report.Type, however we don't want generated.deepcopy to have reference to it.",
+          "description": "ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION|nil) nil == no report.\nmaps to report.Type, however we don't want generated.deepcopy to have reference to it.",
           "type": [
             "string",
             "null"
           ],
           "enum": [
             "JSON",
-            "XML"
+            "XML",
+            "JUNIT-TEST",
+            "JUNIT-STEP",
+            "JUNIT-OPERATION"
           ]
         },
         "reportName": {

pkg/data/schemas/json/configuration-chainsaw-v1alpha2.json

@@ -1747,15 +1747,18 @@
           ],
           "properties": {
             "format": {
-              "description": "ReportFormat determines test report format (JSON|XML).",
+              "description": "ReportFormat determines test report format (JSON|XML|JUNIT-TEST|JUNIT-STEP|JUNIT-OPERATION).",
               "type": [
                 "string",
                 "null"
               ],
               "default": "JSON",
               "enum": [
                 "JSON",
-                "XML"
+                "XML",
+                "JUNIT-TEST",
+                "JUNIT-STEP",
+                "JUNIT-OPERATION"
               ]
             },
             "name": {

pkg/report/junit.go

@@ -8,65 +8,156 @@
 
 	"github.com/jstemmer/go-junit-report/v2/junit"
 	"github.com/kyverno/chainsaw/pkg/model"
+	"go.uber.org/multierr"
 )
 
-// durationInSecondsString is a helper function to convert a start and end time into a string
-// representing the duration in seconds. This is needed by the junit package for generating
-// the JUnit XML report.
 func durationInSecondsString(start, end time.Time) string {
 	duration := end.Sub(start)
 	return fmt.Sprintf("%.6f", duration.Seconds())
 }
 
-// addTestSuite loops through all the operations reports for each directory that is
-// being tested and adds them to the JUnit XML report. This is done by looping through
-// all tests and then looping through all of their steps and all the reports for all steps.
-//
-// The end goal is to have each Test file represented as a TestSuite and each step as a TestCase.
-func addTestSuite(testSuites *junit.Testsuites, report *model.TestReport) {
-	// Loop through all the Tests in the report
-	suite := junit.Testsuite{
-		Name:    report.Name,
-		Package: report.BasePath,
-		Time:    durationInSecondsString(report.StartTime, report.EndTime),
+func saveJUnitTest(report *model.Report, file string) error {
+	testSuites := &junit.Testsuites{
+		Name: report.Name,
+		Time: durationInSecondsString(report.StartTime, report.EndTime),
+	}
+	addTestSuite := func(folder string, tests ...*model.TestReport) {
+		testSuite := junit.Testsuite{
+			Name: folder,
+		}
+		for _, test := range tests {
+			testCase := junit.Testcase{
+				Name: test.Name,
+				Time: durationInSecondsString(test.StartTime, test.EndTime),
+			}
+			if test.Skipped {
+				testCase.Skipped = &junit.Result{}
+			} else {
+				var errs []error
+				for _, step := range test.Steps {
+					for _, operation := range step.Operations {
+						if operation.Err != nil {
+							errs = append(errs, operation.Err)
+						}
+					}
+				}
+				if err := multierr.Combine(errs...); err != nil {
+					testCase.Failure = &junit.Result{
+						Message: err.Error(),
+					}
+				}
+			}
+			testSuite.AddTestcase(testCase)
+		}
+		testSuites.AddSuite(testSuite)
 	}
-	suite.SetTimestamp(report.StartTime)
-	suite.AddProperty("namespace", report.Namespace)
-	if report.Skipped {
-		suite.Skipped = suite.Skipped + 1
+	perFolder := map[string][]*model.TestReport{}
+	for _, test := range report.Tests {
+		perFolder[test.BasePath] = append(perFolder[test.BasePath], test)
+	}
+	for folder, tests := range perFolder {
+		addTestSuite(folder, tests...)
+	}
+	data, err := xml.MarshalIndent(testSuites, "", "  ")
+	if err != nil {
+		return err
 	}
-	for _, report := range report.Steps {
-		testCase := junit.Testcase{
-			Name: report.Name,
-			Time: durationInSecondsString(report.StartTime, report.EndTime),
+	return os.WriteFile(file, data, 0o600)
+}
+
+func saveJUnitStep(report *model.Report, file string) error {
+	testSuites := &junit.Testsuites{
+		Name: report.Name,
+		Time: durationInSecondsString(report.StartTime, report.EndTime),
+	}
+	addTestSuite := func(test *model.TestReport) {
+		testSuite := junit.Testsuite{
+			Name:    test.Name,
+			Package: test.BasePath,
+			Time:    durationInSecondsString(test.StartTime, test.EndTime),
 		}
-		if report.Failed() {
-			testCase.Failure = &junit.Result{}
-			for _, report := range report.Operations {
-				if report.Err != nil {
-					testCase.Failure.Message = report.Err.Error()
-					break
+		testSuite.SetTimestamp(report.StartTime)
+		testSuite.AddProperty("namespace", test.Namespace)
+		if test.Skipped {
+			testCase := junit.Testcase{
+				Name: test.Name,
+				Time: durationInSecondsString(test.StartTime, test.EndTime),
+			}
+			testCase.Skipped = &junit.Result{}
+			testSuite.AddTestcase(testCase)
+		} else {
+			for _, step := range test.Steps {
+				testCase := junit.Testcase{
+					Name: step.Name,
+					Time: durationInSecondsString(step.StartTime, step.EndTime),
+				}
+				var errs []error
+				for _, operation := range step.Operations {
+					if operation.Err != nil {
+						errs = append(errs, operation.Err)
+					}
 				}
+				if err := multierr.Combine(errs...); err != nil {
+					testCase.Failure = &junit.Result{
+						Message: err.Error(),
+					}
+				}
+				testSuite.AddTestcase(testCase)
 			}
 		}
-		suite.AddTestcase(testCase)
+		testSuites.AddSuite(testSuite)
+	}
+	for _, test := range report.Tests {
+		addTestSuite(test)
 	}
-	testSuites.AddSuite(suite)
+	data, err := xml.MarshalIndent(testSuites, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(file, data, 0o600)
 }
 
-// saveJUnit writes the JUnit XML report to disk. The spec is defined here:
-// https://github.com/testmoapp/junitxml
-//
-// This method makes use of https://github.com/jstemmer/go-junit-report to generate the XML.
-func saveJUnit(report *model.Report, file string) error {
-	// Initialize the top-level TestSuites object
+func saveJUnitOperation(report *model.Report, file string) error {
 	testSuites := &junit.Testsuites{
 		Name: report.Name,
 		Time: durationInSecondsString(report.StartTime, report.EndTime),
 	}
-	// Append the individual test suites to the parent
+	addTestSuite := func(test *model.TestReport) {
+		testSuite := junit.Testsuite{
+			Name:    test.Name,
+			Package: test.BasePath,
+			Time:    durationInSecondsString(test.StartTime, test.EndTime),
+		}
+		testSuite.SetTimestamp(report.StartTime)
+		testSuite.AddProperty("namespace", test.Namespace)
+		if test.Skipped {
+			testCase := junit.Testcase{
+				Name: test.Name,
+				Time: durationInSecondsString(test.StartTime, test.EndTime),
+			}
+			testCase.Skipped = &junit.Result{}
+			testSuite.AddTestcase(testCase)
+		} else {
+			for _, step := range test.Steps {
+				for _, operation := range step.Operations {
+					testCase := junit.Testcase{
+						Name:      fmt.Sprintf("%s / %s", step.Name, operation.Name),
+						Classname: string(operation.Type),
+						Time:      durationInSecondsString(operation.StartTime, operation.EndTime),
+					}
+					if err := operation.Err; err != nil {
+						testCase.Failure = &junit.Result{
+							Message: err.Error(),
+						}
+					}
+					testSuite.AddTestcase(testCase)
+				}
+			}
+		}
+		testSuites.AddSuite(testSuite)
+	}
 	for _, test := range report.Tests {
-		addTestSuite(testSuites, test)
+		addTestSuite(test)
 	}
 	data, err := xml.MarshalIndent(testSuites, "", "  ")
 	if err != nil {