New: Ensure code coverage is met

.github/workflows/pre-submit.units.yml

@@ -58,6 +58,10 @@ jobs:
 
           # Build Generic builder.
           go build -mod=vendor ./internal/builders/generic
+      - name: Verify coverage
+        run: |
+              set -euo pipefail
+              make coverage
 
   check-verifier:
     name: verify slsa-verifier is latest

.gitignore

@@ -10,6 +10,7 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+coverage
 
 # Dependency directories (remove the comment below to include it)
 vendor/

Makefile

@@ -1,5 +1,8 @@
 SHELL := /bin/bash
 OUTPUT_FORMAT = $(shell if [ "${GITHUB_ACTIONS}" == "true" ]; then echo "github"; else echo ""; fi)
+TEST_COVERAGE_PERCENTAGE=70
+COVERAGE_THRESHOLD_FILE=coveragethreshold.json
+
 
 .PHONY: help
 help: ## Shows all targets and help from the Makefile (this message).
@@ -30,6 +33,10 @@ unit-test: ## Runs all unit tests.
 	go mod vendor
 	go test -mod=vendor -v ./...
 
+coverage: ## Runs all unit tests and generates a coverage report.
+	echo "Ensuring the code coverage is met"
+	go mod vendor
+	go test -mod=vendor -coverprofile=coverage ./... | go run ./hack/codecoverage/main.go $(COVERAGE_THRESHOLD_FILE) $(TEST_COVERAGE_PERCENTAGE)
 
 ## Linters
 #####################################################################

coveragethreshold.json

@@ -0,0 +1,9 @@
+{
+"github.com/slsa-framework/slsa-github-generator/github":70.4,
+"github.com/slsa-framework/slsa-github-generator/internal/builders/generic": 52.3,
+"github.com/slsa-framework/slsa-github-generator/internal/builders/go":17.1,
+"github.com/slsa-framework/slsa-github-generator/internal/errors":100.0,
+"github.com/slsa-framework/slsa-github-generator/internal/utils":72.1,
+"github.com/slsa-framework/slsa-github-generator/signing/envelope":82.4,
+"github.com/slsa-framework/slsa-github-generator/slsa":54.6
+}

hack/codecoverage/README.md

@@ -0,0 +1,34 @@
+# Go Coverage tool
+
+The goal of the coverage tool is to measure the coverage of the code base for Golang.
+
+## Usage
+Execute the following command to get the coverage and store it in a file:
+1. `go test  -coverprofile=coverage ./... |  run ./hack/codecoverage/main.go coverage.json 70`
+2. The coverage.json contains the percentage of the code coverage that is required to pass for certain packages. This is usually because they don't match the desired coverage.
+```json
+{
+  "github.com/slsa-framework/slsa-github-generator/github":70.4,
+  "github.com/slsa-framework/slsa-github-generator/internal/builders/generic": 52.3,
+  "github.com/slsa-framework/slsa-github-generator/internal/builders/go":17.1,
+  "github.com/slsa-framework/slsa-github-generator/internal/errors":100.0,
+  "github.com/slsa-framework/slsa-github-generator/internal/utils":72.1,
+  "github.com/slsa-framework/slsa-github-generator/signing/envelope":82.4,
+  "github.com/slsa-framework/slsa-github-generator/slsa":54.6
+}
+```
+3. The `COVERAGE_PERCENTAGE` is the percentage of the code coverage that is required to pass for all the packages except the ones that are mentioned in the `THRESHOLD_FILE`.
+4. The coverage tool will fail if the coverage is below the threshold for any package.
+``` shell
+2022/07/29 16:14:41 github.com/slsa-framework/slsa-github-generator/pkg/foo is below the threshold of 71.000000
+exit status 1
+```
+
+### Design choices
+
+1. The coverage tool should not depend on any other tools. It should work of the results from the `go test` command.
+2. Coverage threshold should be configurable for each repository - for example `70%` within the repository.
+3. A setting file should override the coverage threshold for a given package within the repository. `github.com/foo/bar/xyz : 61`
+4. The coverage tool should use native `go` tools and shouldn't depend on external vendors.
+5. The coverage tool should be configurable as part of the PR to fail if the desired threshold is not met.
+6. Contributors should be able to run it locally if desired before doing a PR.

hack/codecoverage/go.mod

@@ -0,0 +1,3 @@
+module github.com/slsa-framework/slsa-github-generator/hack/coverage
+
+go 1.18

hack/codecoverage/main.go

@@ -0,0 +1,91 @@
+// Copyright 2022 SLSA Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		log.Fatal("Usage: <threshold-file> <coverage-percent>")
+	}
+	thresholdFile := os.Args[1]
+	if thresholdFile == "" {
+		log.Fatalf("The thresholdfile cannot be empty.")
+	}
+	thresholdMap, err := parseCoverageThreshold(thresholdFile)
+	if err != nil {
+		log.Fatalf("Error parsing threshold file: %v", err)
+	}
+	coveragePercentage := os.Args[2]
+	if coveragePercentage == "" {
+		log.Fatalf("The coverage percentage cannot be empty.")
+	}
+	coveragePercentageFloat, err := strconv.ParseFloat(coveragePercentage, 64)
+	if err != nil {
+		log.Fatalf("Error parsing coverage percentage: %v", err)
+	}
+	// read stream from stdin
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "coverage: ") {
+			parts := strings.Fields(line)
+			if len(parts) < 5 {
+				continue
+			}
+			percentage, err := strconv.ParseFloat(strings.Trim(parts[4], "%"), 64)
+			if err != nil {
+				log.Fatalf("invalid line: %s", line)
+			}
+			pack := parts[1]
+			if val, ok := thresholdMap[pack]; ok {
+				if percentage < val {
+					log.Fatalf("coverage for %s is below threshold: %f < %f", pack, percentage, val)
+				}
+				continue
+			}
+			// if the package is not in the threshold map, then we check if the coverage is below the threshold
+			if percentage < coveragePercentageFloat {
+				log.Fatalf("coverage for %s is below threshold: %f < %f", pack, percentage, coveragePercentageFloat)
+			}
+		}
+	}
+}
+
+// parseCoverageThreshold parses the threshold file and returns a map.
+func parseCoverageThreshold(fileName string) (map[string]float64, error) {
+	// Here is an example of the threshold file:
+	/*
+		{
+			  "github.com/foo/bar/pkg/cryptoutils": 71.2,
+			}
+	*/
+	f, err := os.Open(fileName)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	thresholdMap := make(map[string]float64)
+	if err := json.NewDecoder(f).Decode(&thresholdMap); err != nil {
+		return nil, err
+	}
+	return thresholdMap, nil
+}