Skip to content

Commit a129335

Browse files
New: Ensure code coverage is met
- This will enable the desired code coverage is met for the project - The coverage is set to 70 to start off with. This setting is in the Makefile - The coveragethreshold.json is an override for packages which have different coverage needs from the global coverage threshold. - The code coverage tool uses standard go tool. Signed-off-by: naveensrinivasan <[email protected]>
1 parent 82a95be commit a129335

File tree

6 files changed

+146
-1
lines changed

6 files changed

+146
-1
lines changed

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
# Output of the go coverage tool, specifically when used with LiteIDE
1212
*.out
13+
coverage
1314

1415
# Dependency directories (remove the comment below to include it)
1516
vendor/

Makefile

+11-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
SHELL := /bin/bash
22
OUTPUT_FORMAT = $(shell if [ "${GITHUB_ACTIONS}" == "true" ]; then echo "github"; else echo ""; fi)
3+
TEST_COVERAGE_PERCENTAGE=70
4+
COVERAGE_THRESHOLD_FILE=coveragethreshold.json
35

4-
.PHONY: help
6+
7+
.PHONY: help coverage
58
help: ## Shows all targets and help from the Makefile (this message).
69
@echo "slsa-github-generator Makefile"
710
@echo "Usage: make [COMMAND]"
@@ -30,6 +33,13 @@ unit-test: ## Runs all unit tests.
3033
go mod vendor
3134
go test -mod=vendor -v ./...
3235

36+
coverage: unit-test ## Runs all unit tests and generates a coverage report.
37+
@echo "Ensuring the code coverage is met"
38+
@go test -mod=vendor -coverprofile=coverage ./... | THRESHOLD_FILE=$(COVERAGE_THRESHOLD_FILE) COVERAGE_PERCENTAGE=$(TEST_COVERAGE_PERCENTAGE) go run ./hack/codecoverage/main.go
39+
40+
@cd .github/actions/detect-workflow
41+
@go mod vendor
42+
@go test -mod=vendor -coverprofile=coverage ./... | THRESHOLD_FILE=$(COVERAGE_THRESHOLD_FILE) COVERAGE_PERCENTAGE=$(TEST_COVERAGE_PERCENTAGE) go run ./hack/codecoverage/main.go
3343

3444
## Linters
3545
#####################################################################

coveragethreshold.json

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"github.com/slsa-framework/slsa-github-generator/github":70.4,
3+
"github.com/slsa-framework/slsa-github-generator/internal/builders/generic": 52.3,
4+
"github.com/slsa-framework/slsa-github-generator/internal/builders/go":17.1,
5+
"github.com/slsa-framework/slsa-github-generator/internal/errors":100.0,
6+
"github.com/slsa-framework/slsa-github-generator/internal/utils":72.1,
7+
"github.com/slsa-framework/slsa-github-generator/signing/envelope":82.4,
8+
"github.com/slsa-framework/slsa-github-generator/slsa":54.6
9+
}

hack/codecoverage/README.md

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Go Coverage tool
2+
3+
The goal of the coverage tool is to measure the coverage of the code base for Golang.
4+
5+
## Usage
6+
Execute the following command to get the coverage and store it in a file:
7+
1. `go test -coverprofile=coverage ./... | THRESHOLD_FILE=./coverage.json COVERAGE_PERCENTAGE=70 go run ./hack/codecoverage/main.go`
8+
2. The `THRESHOLD_FILE` is the path to the file containing the coverage threshold.
9+
3. The THRESHOLD_FILE contains the percentage of the code coverage that is required to pass for certain packages. This is usually because they don't match the desired coverage.
10+
```json
11+
{
12+
"github.com/slsa-framework/slsa-github-generator/github":70.4,
13+
"github.com/slsa-framework/slsa-github-generator/internal/builders/generic": 52.3,
14+
"github.com/slsa-framework/slsa-github-generator/internal/builders/go":17.1,
15+
"github.com/slsa-framework/slsa-github-generator/internal/errors":100.0,
16+
"github.com/slsa-framework/slsa-github-generator/internal/utils":72.1,
17+
"github.com/slsa-framework/slsa-github-generator/signing/envelope":82.4,
18+
"github.com/slsa-framework/slsa-github-generator/slsa":54.6
19+
}
20+
```
21+
3. The `COVERAGE_PERCENTAGE` is the percentage of the code coverage that is required to pass for all the packages except the ones that are mentioned in the `THRESHOLD_FILE`.
22+
4. The coverage tool will fail if the coverage is below the threshold for any package.
23+
``` shell
24+
2022/07/29 16:14:41 github.com/slsa-framework/slsa-github-generator/pkg/foo is below the threshold of 71.000000
25+
exit status 1
26+
```
27+
28+
### Design choices
29+
30+
1. The coverage tool should not depend on any other tools. It should work of the results from the `go test` command.
31+
2. Coverage threshold should be configurable for each repository - for example `70%` within the repository.
32+
3. A setting file should override the coverage threshold for a given package within the repository. `github.com/foo/bar/xyz : 61`
33+
4. The coverage tool should use native `go` tools and shouldn't depend on external vendors.
34+
5. The coverage tool should be configurable as part of the PR to fail if the desired threshold is not met.
35+
6. Contributors should be able to run it locally if desired before doing a PR.

hack/codecoverage/go.mod

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
module github.com/slsa-framework/slsa-github-generator/hack/coverage
2+
3+
go 1.18

hack/codecoverage/main.go

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Copyright 2022 SLSA Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
package main
15+
16+
import (
17+
"bufio"
18+
"encoding/json"
19+
"log"
20+
"os"
21+
"strconv"
22+
"strings"
23+
)
24+
25+
func main() {
26+
thresholdFile := os.Getenv("THRESHOLD_FILE")
27+
if thresholdFile == "" {
28+
log.Fatalf("THRESHOLD_FILE environment variable is not set")
29+
}
30+
thresholdMap, err := parseCoverageThreshold(thresholdFile)
31+
if err != nil {
32+
log.Fatalf("Error parsing threshold file: %v", err)
33+
}
34+
coveragePercentage := os.Getenv("COVERAGE_PERCENTAGE")
35+
if coveragePercentage == "" {
36+
log.Fatalf("COVERAGE_PERCENTAGE environment variable is not set")
37+
}
38+
coveragePercentageFloat, err := strconv.ParseFloat(coveragePercentage, 32)
39+
if err != nil {
40+
log.Fatalf("Error parsing coverage percentage: %v", err)
41+
}
42+
// read stream from stdin
43+
scanner := bufio.NewScanner(os.Stdin)
44+
for scanner.Scan() {
45+
line := scanner.Text()
46+
if strings.Contains(line, "coverage: ") {
47+
parts := strings.Fields(line)
48+
if len(parts) < 5 {
49+
continue
50+
}
51+
percentage, err := strconv.ParseFloat(strings.Trim(parts[4], "%"), 32)
52+
if err != nil {
53+
log.Fatalf("invalid line: %s", line)
54+
}
55+
pack := parts[1]
56+
if val, ok := thresholdMap[pack]; !ok {
57+
if float32(int(percentage*100)/100) < float32(int(coveragePercentageFloat*100)/100) {
58+
log.Fatalf("coverage for %s is below threshold: %f < %f", pack, percentage, coveragePercentageFloat)
59+
}
60+
} else {
61+
if float32(int(percentage*100)/100) < float32(int(val*100)/100) {
62+
log.Fatalf("coverage for %s is below threshold: %f < %f", pack, percentage, val)
63+
}
64+
}
65+
}
66+
}
67+
}
68+
69+
// parseCoverageThreshold parses the threshold file and returns a map.
70+
func parseCoverageThreshold(fileName string) (map[string]float64, error) {
71+
// Here is an example of the threshold file:
72+
/*
73+
{
74+
"github.com/foo/bar/pkg/cryptoutils": 71.2,
75+
}
76+
*/
77+
f, err := os.Open(fileName)
78+
if err != nil {
79+
return nil, err
80+
}
81+
defer f.Close()
82+
thresholdMap := make(map[string]float64)
83+
if err := json.NewDecoder(f).Decode(&thresholdMap); err != nil {
84+
return nil, err
85+
}
86+
return thresholdMap, nil
87+
}

0 commit comments

Comments
 (0)