Skip to content

Commit 19434ce

Browse files
eddycharlyeddycharly
authored
refactor: move reports management into caller (#2230)
* refactor: move reports management into caller Signed-off-by: Charles-Edouard Brétéché <[email protected]> * fix tests Signed-off-by: Charles-Edouard Brétéché <[email protected]> --------- Signed-off-by: Charles-Edouard Brétéché <[email protected]>
1 parent 90d2e2c commit 19434ce

13 files changed

+74
-84
lines changed

‎pkg/commands/test/command.go

+15-7
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/kyverno/chainsaw/pkg/discovery"
1313
"github.com/kyverno/chainsaw/pkg/loaders/config"
1414
"github.com/kyverno/chainsaw/pkg/loaders/values"
15+
"github.com/kyverno/chainsaw/pkg/report"
1516
"github.com/kyverno/chainsaw/pkg/runner"
1617
runnerflags "github.com/kyverno/chainsaw/pkg/runner/flags"
1718
flagutils "github.com/kyverno/chainsaw/pkg/utils/flag"
@@ -351,16 +352,23 @@ func Command() *cobra.Command {
351352
if err := runnerflags.SetupFlags(configuration.Spec); err != nil {
352353
return err
353354
}
354-
summary, err := runner.Run(ctx, configuration.Spec, tc, testToRun...)
355-
if summary != nil {
356-
fmt.Fprintln(stdOut, "Tests Summary...")
357-
fmt.Fprintln(stdOut, "- Passed tests", summary.Passed())
358-
fmt.Fprintln(stdOut, "- Failed tests", summary.Failed())
359-
fmt.Fprintln(stdOut, "- Skipped tests", summary.Skipped())
355+
err = runner.Run(ctx, configuration.Spec, tc, testToRun...)
356+
fmt.Fprintln(stdOut, "Tests Summary...")
357+
fmt.Fprintln(stdOut, "- Passed tests", tc.Passed())
358+
fmt.Fprintln(stdOut, "- Failed tests", tc.Failed())
359+
fmt.Fprintln(stdOut, "- Skipped tests", tc.Skipped())
360+
// process report
361+
if err == nil {
362+
if configuration.Spec.Report != nil && configuration.Spec.Report.Format != "" {
363+
fmt.Fprintln(stdOut, "Saving report...")
364+
if err := report.Save(tc.Report, configuration.Spec.Report.Format, configuration.Spec.Report.Path, configuration.Spec.Report.Name); err != nil {
365+
return err
366+
}
367+
}
360368
}
361369
if err != nil {
362370
fmt.Fprintln(stdOut, "Done with error.")
363-
} else if summary != nil && summary.Failed() > 0 {
371+
} else if tc.Failed() > 0 {
364372
fmt.Fprintln(stdOut, "Done with failures.")
365373
err = errors.New("some tests failed")
366374
} else {

‎pkg/commands/test/command_test.go

+5
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
)
1212

1313
func TestChainsawCommand(t *testing.T) {
14+
path := "../../../.temp"
1415
basePath := "../../../testdata/commands/test"
1516
tests := []struct {
1617
name string
@@ -116,6 +117,8 @@ func TestChainsawCommand(t *testing.T) {
116117
args: []string{
117118
"--config",
118119
filepath.Join(basePath, "config/config_all_fields.yaml"),
120+
"--report-path",
121+
path,
119122
},
120123
wantErr: false,
121124
out: filepath.Join(basePath, "config_all_fields.txt"),
@@ -135,6 +138,8 @@ func TestChainsawCommand(t *testing.T) {
135138
"--parallel=24",
136139
"--repeat-count=12",
137140
"--report-format=XML",
141+
"--report-path",
142+
path,
138143
"--report-name=foo",
139144
"--namespace=bar",
140145
"--full-name=true",

‎pkg/runner/internal/test_deps_test.go

+5
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,11 @@ func TestTestDeps_MatchString(t *testing.T) {
8080
}
8181
}
8282

83+
func TestTestDeps_SetPanicOnExit0(t *testing.T) {
84+
d := &TestDeps{}
85+
d.SetPanicOnExit0(true)
86+
}
87+
8388
func TestTestDeps_StartCPUProfile(t *testing.T) {
8489
d := &TestDeps{}
8590
assert.NoError(t, d.StartCPUProfile(nil))

‎pkg/runner/runner.go

+6-13
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"github.com/kyverno/chainsaw/pkg/discovery"
1010
"github.com/kyverno/chainsaw/pkg/engine/clusters"
1111
"github.com/kyverno/chainsaw/pkg/model"
12-
"github.com/kyverno/chainsaw/pkg/report"
1312
enginecontext "github.com/kyverno/chainsaw/pkg/runner/context"
1413
"github.com/kyverno/chainsaw/pkg/runner/internal"
1514
"github.com/kyverno/chainsaw/pkg/testing"
@@ -18,7 +17,7 @@ import (
1817
)
1918

2019
type Runner interface {
21-
Run(context.Context, model.Configuration, enginecontext.TestContext, ...discovery.Test) (model.SummaryResult, error)
20+
Run(context.Context, model.Configuration, enginecontext.TestContext, ...discovery.Test) error
2221
}
2322

2423
func New(clock clock.PassiveClock, onFailure func()) Runner {
@@ -34,14 +33,14 @@ type runner struct {
3433
deps *internal.TestDeps
3534
}
3635

37-
func (r *runner) Run(ctx context.Context, config model.Configuration, tc enginecontext.TestContext, tests ...discovery.Test) (model.SummaryResult, error) {
36+
func (r *runner) Run(ctx context.Context, config model.Configuration, tc enginecontext.TestContext, tests ...discovery.Test) error {
3837
return r.run(ctx, nil, config, tc, tests...)
3938
}
4039

41-
func (r *runner) run(ctx context.Context, m mainstart, config model.Configuration, tc enginecontext.TestContext, tests ...discovery.Test) (model.SummaryResult, error) {
40+
func (r *runner) run(ctx context.Context, m mainstart, config model.Configuration, tc enginecontext.TestContext, tests ...discovery.Test) error {
4241
// sanity check
4342
if len(tests) == 0 {
44-
return nil, nil
43+
return nil
4544
}
4645
internalTests := []testing.InternalTest{{
4746
Name: "chainsaw",
@@ -66,16 +65,10 @@ func (r *runner) run(ctx context.Context, m mainstart, config model.Configuratio
6665
// In our case, we consider an error only when running the tests was not possible.
6766
// For now, the case where some of the tests failed will be covered by the summary.
6867
if code := m.Run(); code > 1 {
69-
return nil, fmt.Errorf("testing framework exited with non zero code %d", code)
68+
return fmt.Errorf("testing framework exited with non zero code %d", code)
7069
}
7170
tc.Report.EndTime = time.Now()
72-
// TODO: move to the caller
73-
if config.Report != nil && config.Report.Format != "" {
74-
if err := report.Save(tc.Report, config.Report.Format, config.Report.Path, config.Report.Name); err != nil {
75-
return tc, err
76-
}
77-
}
78-
return tc, nil
71+
return nil
7972
}
8073

8174
func (r *runner) onFail() {

‎pkg/runner/runner_test.go

+7-64
Original file line numberDiff line numberDiff line change
@@ -94,19 +94,16 @@ func Test_runner_Run(t *testing.T) {
9494
// to allow unit tests to work
9595
assert.NoError(t, flags.SetupFlags(tt.config))
9696
assert.NoError(t, flag.Set("test.testlogfile", ""))
97-
got, err := r.Run(context.TODO(), tt.config, tt.tc, tt.tests...)
97+
err := r.Run(context.TODO(), tt.config, tt.tc, tt.tests...)
9898
if tt.wantErr {
9999
assert.Error(t, err)
100100
} else {
101101
assert.NoError(t, err)
102102
}
103-
if tt.want == nil {
104-
assert.Nil(t, got)
105-
} else {
106-
assert.NotNil(t, got)
107-
assert.Equal(t, tt.want.failed, got.Failed())
108-
assert.Equal(t, tt.want.passed, got.Passed())
109-
assert.Equal(t, tt.want.skipped, got.Skipped())
103+
if tt.want != nil {
104+
assert.Equal(t, tt.want.failed, tc.Failed())
105+
assert.Equal(t, tt.want.passed, tc.Passed())
106+
assert.Equal(t, tt.want.skipped, tc.Skipped())
110107
}
111108
})
112109
}
@@ -158,7 +155,7 @@ func Test_runner_run(t *testing.T) {
158155
r := &runner{
159156
clock: clock.RealClock{},
160157
}
161-
_, err := r.run(context.TODO(), tt.m, model.Configuration{}, enginecontext.EmptyContext(), tt.tests...)
158+
err := r.run(context.TODO(), tt.m, model.Configuration{}, enginecontext.EmptyContext(), tt.tests...)
162159
if tt.wantErr {
163160
assert.Error(t, err)
164161
} else {
@@ -229,17 +226,6 @@ func TestRun(t *testing.T) {
229226
},
230227
restConfig: nil,
231228
wantErr: false,
232-
}, {
233-
name: "Zero Tests with JSON Report",
234-
tests: []discovery.Test{},
235-
config: model.Configuration{
236-
Timeouts: config.Spec.Timeouts,
237-
Report: &v1alpha2.ReportOptions{
238-
Format: v1alpha2.JSONFormat,
239-
},
240-
},
241-
restConfig: &rest.Config{},
242-
wantErr: false,
243229
}, {
244230
name: "Success Case with 1 Test",
245231
tests: []discovery.Test{
@@ -270,49 +256,6 @@ func TestRun(t *testing.T) {
270256
restConfig: &rest.Config{},
271257
mockReturn: 2,
272258
wantErr: true,
273-
}, {
274-
name: "Success Case with 1 Test with XML Report",
275-
tests: []discovery.Test{
276-
{
277-
Err: nil,
278-
Test: &model.Test{
279-
ObjectMeta: metav1.ObjectMeta{
280-
Name: "test1",
281-
},
282-
},
283-
},
284-
},
285-
config: model.Configuration{
286-
Timeouts: config.Spec.Timeouts,
287-
Report: &v1alpha2.ReportOptions{
288-
Format: v1alpha2.XMLFormat,
289-
Name: "chainsaw",
290-
},
291-
},
292-
restConfig: &rest.Config{},
293-
mockReturn: 0,
294-
wantErr: false,
295-
}, {
296-
name: "Error in saving Report",
297-
tests: []discovery.Test{
298-
{
299-
Err: nil,
300-
Test: &model.Test{
301-
ObjectMeta: metav1.ObjectMeta{
302-
Name: "test1",
303-
},
304-
},
305-
},
306-
},
307-
config: model.Configuration{
308-
Timeouts: config.Spec.Timeouts,
309-
Report: &v1alpha2.ReportOptions{
310-
Format: "abc",
311-
},
312-
},
313-
restConfig: &rest.Config{},
314-
mockReturn: 0,
315-
wantErr: true,
316259
}}
317260
for _, tt := range tests {
318261
t.Run(tt.name, func(t *testing.T) {
@@ -325,7 +268,7 @@ func TestRun(t *testing.T) {
325268
ctx := context.TODO()
326269
tc, err := InitContext(tt.config, tt.restConfig, nil)
327270
assert.NoError(t, err)
328-
_, err = runner.run(ctx, mockMainStart, tt.config, tc, tt.tests...)
271+
err = runner.run(ctx, mockMainStart, tt.config, tc, tt.tests...)
329272
if tt.wantErr {
330273
assert.Error(t, err)
331274
} else {

‎testdata/commands/test/all_flags.txt

+6
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ Loading default configuration...
66
- FailFast false
77
- ReportFormat 'XML'
88
- ReportName 'foo'
9+
- ReportPath '../../../.temp'
910
- Namespace 'bar'
1011
- FullName true
1112
- IncludeTestRegex '^.*$'
@@ -26,4 +27,9 @@ Loading default configuration...
2627
Loading tests...
2728
Loading values...
2829
Running tests...
30+
Tests Summary...
31+
- Passed tests 0
32+
- Failed tests 0
33+
- Skipped tests 0
34+
Saving report...
2935
Done.

‎testdata/commands/test/config_all_fields.txt

+6
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ Loading config (../../../testdata/commands/test/config/config_all_fields.yaml)..
66
- FailFast true
77
- ReportFormat 'JSON'
88
- ReportName 'custom-chainsaw-report'
9+
- ReportPath '../../../.temp'
910
- Namespace 'test-namespace'
1011
- FullName true
1112
- IncludeTestRegex '^include-.*'
@@ -24,4 +25,9 @@ Loading config (../../../testdata/commands/test/config/config_all_fields.yaml)..
2425
Loading tests...
2526
Loading values...
2627
Running tests...
28+
Tests Summary...
29+
- Passed tests 0
30+
- Failed tests 0
31+
- Skipped tests 0
32+
Saving report...
2733
Done.

‎testdata/commands/test/default.txt

+4
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,8 @@ Loading default configuration...
2121
Loading tests...
2222
Loading values...
2323
Running tests...
24+
Tests Summary...
25+
- Passed tests 0
26+
- Failed tests 0
27+
- Skipped tests 0
2428
Done.

‎testdata/commands/test/with_regex.txt

+4
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,8 @@ Loading default configuration...
2121
Loading tests...
2222
Loading values...
2323
Running tests...
24+
Tests Summary...
25+
- Passed tests 0
26+
- Failed tests 0
27+
- Skipped tests 0
2428
Done.

‎testdata/commands/test/with_repeat_count.txt

+4
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,8 @@ Loading default configuration...
2222
Loading tests...
2323
Loading values...
2424
Running tests...
25+
Tests Summary...
26+
- Passed tests 0
27+
- Failed tests 0
28+
- Skipped tests 0
2529
Done.

‎testdata/commands/test/with_suppress.txt

+4
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,8 @@ Loading default configuration...
2424
Loading tests...
2525
Loading values...
2626
Running tests...
27+
Tests Summary...
28+
- Passed tests 0
29+
- Failed tests 0
30+
- Skipped tests 0
2731
Done.

‎testdata/commands/test/with_test_dirs.txt

+4
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,8 @@ Loading default configuration...
2121
Loading tests...
2222
Loading values...
2323
Running tests...
24+
Tests Summary...
25+
- Passed tests 0
26+
- Failed tests 0
27+
- Skipped tests 0
2428
Done.

‎testdata/commands/test/with_timeout.txt

+4
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,8 @@ Loading default configuration...
2121
Loading tests...
2222
Loading values...
2323
Running tests...
24+
Tests Summary...
25+
- Passed tests 0
26+
- Failed tests 0
27+
- Skipped tests 0
2428
Done.

0 commit comments

Comments
 (0)